sensemaking 0.23.0 → 0.24.0

package/dist/cjs/store/duckdb/lexical.js

@@ -1,5 +1,5 @@
 // D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /
-// unspaced scripts, JS excerpts for every doc (no snippet()).
+// unspaced scripts, JS snippets for every doc (no snippet()).
 "use strict";
 Object.defineProperty(exports, "__esModule", {
     value: true
@@ -23,6 +23,7 @@ _export(exports, {
 });
 var _errorsts = require("../../errors.js");
 var _segmentts = require("../../text/segment.js");
+var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -240,50 +241,102 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
+// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of
+// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.
+var FTS_STALE_META_KEY = 'fts_stale';
 // One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
 // stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
 var ftsState = new WeakMap();
 function stateFor(conn) {
-    var state = ftsState.get(conn);
-    if (!state) {
-        state = {
-            stale: true
-        };
-        ftsState.set(conn, state);
-    }
-    return state;
+    return _async_to_generator(function() {
+        var state, _tmp;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    state = ftsState.get(conn);
+                    if (!!state) return [
+                        3,
+                        2
+                    ];
+                    _tmp = {};
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, FTS_STALE_META_KEY)
+                    ];
+                case 1:
+                    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that
+                    // never built an index, or one a prior process marked stale and never got to clear.
+                    state = (_tmp.stale = _state.sent() !== '0', _tmp);
+                    ftsState.set(conn, state);
+                    _state.label = 2;
+                case 2:
+                    return [
+                        2,
+                        state
+                    ];
+            }
+        });
+    })();
 }
 function markContentStale(conn) {
-    stateFor(conn).stale = true;
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    ftsState.set(conn, {
+                        stale: true
+                    });
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, FTS_STALE_META_KEY, '1')
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
 }
-// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
-// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
+// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,
+// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.
 function splitTerms(terms) {
     var words = [];
+    var bareWords = [];
+    var phraseWords = [];
     var substrings = [];
+    var phrases = [];
+    var emptyPhrase = false;
     var withoutPhrases = terms.replace(/"([^"]*)"/g, function(_m, inner) {
         var phrase = inner.trim();
-        if (phrase.length === 0) return ' ';
-        substrings.push(phrase);
-        if (!(0, _segmentts.hasUnspacedRun)(phrase)) {
-            var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+        var phraseTokens = (0, _segmentts.searchTokens)(phrase);
+        if (phraseTokens.length === 0) {
+            emptyPhrase = true;
+            return ' ';
+        }
+        phrases.push(phraseTokens);
+        var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+        try {
+            for(var _iterator = phraseTokens[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                var token = _step.value;
+                if (token.unspaced) substrings.push(token.text);
+                else {
+                    words.push(token.text);
+                    phraseWords.push(token.text);
+                }
+            }
+        } catch (err) {
+            _didIteratorError = true;
+            _iteratorError = err;
+        } finally{
             try {
-                for(var _iterator = phrase.split(/\s+/)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                    var w = _step.value;
-                    if (w.length > 0) words.push(w);
+                if (!_iteratorNormalCompletion && _iterator.return != null) {
+                    _iterator.return();
                 }
-            } catch (err) {
-                _didIteratorError = true;
-                _iteratorError = err;
             } finally{
-                try {
-                    if (!_iteratorNormalCompletion && _iterator.return != null) {
-                        _iterator.return();
-                    }
-                } finally{
-                    if (_didIteratorError) {
-                        throw _iteratorError;
-                    }
+                if (_didIteratorError) {
+                    throw _iteratorError;
                 }
             }
         }
@@ -294,8 +347,30 @@ function splitTerms(terms) {
         for(var _iterator = withoutPhrases.split(/\s+/)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var tok = _step.value;
             if (tok.length === 0) continue;
-            if ((0, _segmentts.hasUnspacedRun)(tok)) substrings.push(tok);
-            else words.push(tok);
+            var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+            try {
+                for(var _iterator1 = (0, _segmentts.searchTokens)(tok)[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                    var token = _step1.value;
+                    if (token.unspaced) substrings.push(token.text);
+                    else {
+                        words.push(token.text);
+                        bareWords.push(token.text);
+                    }
+                }
+            } catch (err) {
+                _didIteratorError1 = true;
+                _iteratorError1 = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                        _iterator1.return();
+                    }
+                } finally{
+                    if (_didIteratorError1) {
+                        throw _iteratorError1;
+                    }
+                }
+            }
         }
     } catch (err) {
         _didIteratorError = true;
@@ -313,11 +388,15 @@ function splitTerms(terms) {
     }
     return {
         words: words,
-        substrings: substrings
+        bareWords: bareWords,
+        phraseWords: phraseWords,
+        substrings: substrings,
+        phrases: phrases,
+        emptyPhrase: emptyPhrase
     };
 }
-// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only
-// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).
+// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays
+// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.
 function ensureFtsFresh(conn, state) {
     return _async_to_generator(function() {
         return _ts_generator(this, function(_state) {
@@ -346,6 +425,14 @@ function ensureFtsFresh(conn, state) {
                                                 conn.exec("PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)")
                                             ];
                                         case 1:
+                                            _state.sent();
+                                            // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant
+                                            // rebuild next time), never a cleared mark serving a half-built index as fresh.
+                                            return [
+                                                4,
+                                                (0, _sharedts.setMeta)(conn, FTS_STALE_META_KEY, '0')
+                                            ];
+                                        case 2:
                                             _state.sent();
                                             return [
                                                 2
@@ -367,13 +454,13 @@ function ensureFtsFresh(conn, state) {
 }
 // Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
 // score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
-function buildScoreAndGate(words, substrings) {
+function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
     var scoreParts = [];
     var gateParts = [];
     var scoreParams = [];
     var gateParams = [];
     if (words.length > 0) {
-        var wordQuery = words.join(' ');
+        var wordQuery = words.map(_segmentts.foldForSearch).join(' ');
         var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
         try {
             for(var _iterator = FIELDS[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
@@ -395,38 +482,63 @@ function buildScoreAndGate(words, substrings) {
                 }
             }
         }
-        gateParts.push('fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL');
-        gateParams.push(wordQuery);
+        if (bareWords.length > 0) {
+            var nativeGate = 'fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL';
+            gateParts.push(nativeGate);
+            gateParams.push(bareWords.map(_segmentts.foldForSearch).join(' '));
+        }
+        if (phraseWords.length > 0) {
+            var _gateParams;
+            var nativeGate1 = 'fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL';
+            var fallback = phraseWords.map(function(_word) {
+                return "(".concat(FIELDS.map(function(field) {
+                    return "contains(strip_accents(lower(content.".concat(field, ")), ?)");
+                }).join(' OR '), ")");
+            }).join(' AND ');
+            gateParts.push("(".concat(nativeGate1, " OR (").concat(fallback, "))"));
+            gateParams.push(phraseWords.map(_segmentts.foldForSearch).join(' '));
+            (_gateParams = gateParams).push.apply(_gateParams, _to_consumable_array(phraseWords.flatMap(function(word) {
+                return FIELDS.map(function() {
+                    return (0, _segmentts.foldForSearch)(word);
+                });
+            })));
+        }
     }
     var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
     try {
-        for(var _iterator1 = substrings[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+        var _loop = function() {
             var raw = _step1.value;
+            var _gateParams;
             var needle = raw.toLowerCase();
-            var _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+            var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
             try {
-                for(var _iterator2 = FIELDS[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true){
-                    var field1 = _step2.value;
-                    scoreParts.push("".concat(FIELD_WEIGHT[field1], ".0 * CAST(contains(lower(content.").concat(field1, "), ?) AS INTEGER)"));
-                    scoreParams.push(needle);
+                for(var _iterator = FIELDS[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                    var field = _step.value;
+                    scoreParts.push("".concat(FIELD_WEIGHT[field], ".0 * CAST(contains(strip_accents(lower(content.").concat(field, ")), ?) AS INTEGER)"));
+                    scoreParams.push((0, _segmentts.foldForSearch)(needle));
                 }
             } catch (err) {
-                _didIteratorError2 = true;
-                _iteratorError2 = err;
+                _didIteratorError = true;
+                _iteratorError = err;
             } finally{
                 try {
-                    if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
-                        _iterator2.return();
+                    if (!_iteratorNormalCompletion && _iterator.return != null) {
+                        _iterator.return();
                     }
                 } finally{
-                    if (_didIteratorError2) {
-                        throw _iteratorError2;
+                    if (_didIteratorError) {
+                        throw _iteratorError;
                     }
                 }
             }
-            gateParts.push("contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)");
-            gateParams.push(needle);
-        }
+            gateParts.push("(".concat(FIELDS.map(function(field) {
+                return "contains(strip_accents(lower(content.".concat(field, ")), ?)");
+            }).join(' OR '), ")"));
+            (_gateParams = gateParams).push.apply(_gateParams, _to_consumable_array(FIELDS.map(function() {
+                return (0, _segmentts.foldForSearch)(needle);
+            })));
+        };
+        for(var _iterator1 = substrings[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true)_loop();
     } catch (err) {
         _didIteratorError1 = true;
         _iteratorError1 = err;
@@ -443,13 +555,13 @@ function buildScoreAndGate(words, substrings) {
     }
     return {
         scoreSql: scoreParts.join(' + '),
-        gateSql: gateParts.join(' AND '),
+        gateSql: gateParts.join(' AND ') || 'TRUE',
         params: _to_consumable_array(scoreParams).concat(_to_consumable_array(gateParams))
     };
 }
 function queryLexical(conn, terms, opts, state) {
     return _async_to_generator(function() {
-        var _stmt, unsupported, whereJoin, whereCond, scopeCond, limit, _splitTerms, words, substrings, _buildScoreAndGate, scoreSql, gateSql, params, sql, stmt;
+        var _stmt, unsupported, whereJoin, whereCond, scopeCond, limit, _splitTerms, words, bareWords, phraseWords, substrings, phrases, emptyPhrase, _buildScoreAndGate, scoreSql, gateSql, params, selectFields, sqlLimit, sql, stmt, rows, matches;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -458,8 +570,8 @@ function queryLexical(conn, terms, opts, state) {
                         throw new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "duckdb" does not implement FTS5\'s '.concat(unsupported.label, ' ("').concat(unsupported.token, '") in this build; rephrase "').concat(terms.trim(), '" without it, or set "store" to "sqlite" in this tree\'s config to search it as written'));
                     }
                     whereJoin = opts.whereJoin, whereCond = opts.whereCond, scopeCond = opts.scopeCond, limit = opts.limit;
-                    _splitTerms = splitTerms(terms), words = _splitTerms.words, substrings = _splitTerms.substrings;
-                    if (words.length === 0 && substrings.length === 0) return [
+                    _splitTerms = splitTerms(terms), words = _splitTerms.words, bareWords = _splitTerms.bareWords, phraseWords = _splitTerms.phraseWords, substrings = _splitTerms.substrings, phrases = _splitTerms.phrases, emptyPhrase = _splitTerms.emptyPhrase;
+                    if (emptyPhrase || words.length === 0 && substrings.length === 0) return [
                         2,
                         []
                     ];
@@ -475,8 +587,10 @@ function queryLexical(conn, terms, opts, state) {
                     _state.sent();
                     _state.label = 2;
                 case 2:
-                    _buildScoreAndGate = buildScoreAndGate(words, substrings), scoreSql = _buildScoreAndGate.scoreSql, gateSql = _buildScoreAndGate.gateSql, params = _buildScoreAndGate.params;
-                    sql = 'SELECT path, NULL AS hit FROM (\n    SELECT content."path" AS path, ('.concat(scoreSql, ") AS score\n    FROM content\n    ").concat(whereJoin, "\n    WHERE ").concat(gateSql, "\n    ").concat(whereCond, " ").concat(scopeCond, "\n  ) sq ORDER BY score DESC, path LIMIT ?");
+                    _buildScoreAndGate = buildScoreAndGate(words, bareWords, phraseWords, substrings), scoreSql = _buildScoreAndGate.scoreSql, gateSql = _buildScoreAndGate.gateSql, params = _buildScoreAndGate.params;
+                    selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
+                    sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';
+                    sql = "SELECT path, score".concat(phrases.length > 0 ? ', title, summary, text' : '', ' FROM (\n    SELECT content."path" AS path, (').concat(scoreSql, ") AS score").concat(selectFields, "\n    FROM content\n    ").concat(whereJoin, "\n    WHERE ").concat(gateSql, "\n    ").concat(whereCond, " ").concat(scopeCond, "\n    ) sq ORDER BY score DESC, path").concat(sqlLimit);
                     return [
                         4,
                         conn.prepare(sql)
@@ -485,14 +599,28 @@ function queryLexical(conn, terms, opts, state) {
                     stmt = _state.sent();
                     return [
                         4,
-                        (_stmt = stmt).all.apply(_stmt, _to_consumable_array(params).concat([
+                        (_stmt = stmt).all.apply(_stmt, _to_consumable_array(params).concat(_to_consumable_array(phrases.length > 0 ? [] : [
                             limit
-                        ]))
+                        ])))
                     ];
                 case 4:
+                    rows = _state.sent();
+                    matches = phrases.length === 0 ? rows : rows.filter(function(row) {
+                        return phrases.every(function(phrase) {
+                            return FIELDS.some(function(field) {
+                                var _row_field;
+                                return (0, _segmentts.matchesSearchPhrase)((_row_field = row[field]) !== null && _row_field !== void 0 ? _row_field : '', phrase);
+                            });
+                        });
+                    });
                     return [
                         2,
-                        _state.sent()
+                        matches.slice(0, limit).map(function(param) {
+                            var path = param.path;
+                            return {
+                                path: path
+                            };
+                        })
                     ];
             }
         });
@@ -501,7 +629,30 @@ function queryLexical(conn, terms, opts, state) {
 function createLexicalIndex(conn) {
     return {
         query: function query(terms, opts) {
-            return queryLexical(conn, terms, opts, stateFor(conn));
+            return _async_to_generator(function() {
+                var _tmp;
+                return _ts_generator(this, function(_state) {
+                    switch(_state.label){
+                        case 0:
+                            _tmp = [
+                                conn,
+                                terms,
+                                opts
+                            ];
+                            return [
+                                4,
+                                stateFor(conn)
+                            ];
+                        case 1:
+                            return [
+                                2,
+                                queryLexical.apply(void 0, _tmp.concat([
+                                    _state.sent()
+                                ]))
+                            ];
+                    }
+                });
+            })();
         },
         markStale: function markStale() {
             return markContentStale(conn);

package/dist/cjs/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nfunction stateFor(conn: Connection): FtsIndexState {\n  let state = ftsState.get(conn);\n  if (!state) {\n    state = { stale: true };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`, so the next lexical query\n// rebuilds instead of searching stale content.\nexport function markContentStale(conn: Connection): void {\n  stateFor(conn).stale = true;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches\n// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","ftsState","WeakMap","stateFor","conn","state","get","stale","set","splitTerms","words","substrings","_m","inner","phrase","length","push","hasUnspacedRun","split","w","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAgJ9CA;eAAAA;;QA/FAC;eAAAA;;QAsEMC;eAAAA;;;wBAtHK;yBACI;6BACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAASC,SAASC,IAAgB;IAChC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;QAAK;QACtBN,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAIO,SAAStB,iBAAiBqB,IAAgB;IAC/CD,SAASC,MAAMG,KAAK,GAAG;AACzB;AAEA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASE,WAAWf,KAAa;IAC/B,IAAMgB,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMhB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACgB,IAAIC;QACtD,IAAMC,SAASD,MAAMb,IAAI;QACzB,IAAIc,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACG,IAAAA,yBAAc,EAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACjG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaxB,eAAeuB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIE,IAAAA,yBAAc,EAACG,MAAMT,WAAWK,IAAI,CAACI;iBACpCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,SAAeU,eAAejB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAME,KAAK,EAAE;;;oBAClB;;wBAAMH,KAAKkB,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACnB,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKkB,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBAJA;oBAKAjB,MAAME,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASiB,kBAAkBd,KAAe,EAAEC,UAAoB;IAC9D,IAAMc,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIlB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMc,YAAYnB,MAAMoB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAe7C,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAM8C,QAAN;gBACHN,WAAWT,IAAI,CAAC,AAAC,GAAgGe,OAA9F7C,YAAY,CAAC6C,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYX,IAAI,CAACa;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUV,IAAI,CAAC;QACfY,WAAWZ,IAAI,CAACa;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAalB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMqB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAejD,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAM8C,SAAN;oBACHN,WAAWT,IAAI,CAAC,AAAC,GAAyDe,OAAvD7C,YAAY,CAAC6C,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYX,IAAI,CAACiB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUV,IAAI,CAAC;YACfY,WAAWZ,IAAI,CAACiB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAIO,SAAe5C,aAAaoB,IAAgB,EAAEV,KAAa,EAAE4C,IAAyB,EAAEjC,KAAoB;;YAoBnGkC,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXnC,aAAtBC,OAAOC,YAKuBa,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc/C,oBAAoBC;oBACxC,IAAI8C,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAYjD,KAAK,EAAC,OAAqDG,OAAhD8C,YAAYzC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQyC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXnC,cAAAA,WAAWf,QAAjCgB,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAejB,MAAMC;;;oBAA3B;;;oBAEgBmB,qBAAAA,kBAAkBd,OAAOC,aAAvDwB,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMvC,KAAK2C,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAIO,SAAS9D,mBAAmBsB,IAAgB;IACjD,OAAO;QACL6C,OAAO,SAAPA,MAAQvD,OAAO4C;mBAAStD,aAAaoB,MAAMV,OAAO4C,MAAMnC,SAASC;;QACjE8C,WAAW,SAAXA;mBAAiBnE,iBAAiBqB;;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0' };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","getMeta","stale","set","setMeta","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","searchTokens","length","push","unspaced","split","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","foldForSearch","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","matches","SenseError","prepare","all","filter","row","every","some","matchesSearchPhrase","slice","path","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QA4L9CA;eAAAA;;QApIMC;eAAAA;;QAuGAC;eAAAA;;;wBA9JK;yBACwD;wBAClD;6BACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,kGAAkG;AAClG,kGAAkG;AAClG,IAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAAeC,SAASC,IAAgB;;YAClCC;;;;oBAAAA,QAAQJ,SAASK,GAAG,CAACF;yBACrB,CAACC,OAAD;;;;;oBAGgB;;wBAAME,IAAAA,iBAAO,EAACH,MAAMJ;;;oBAFtC,2FAA2F;oBAC3F,oFAAoF;oBACpFK,cAAUG,QAAO,AAAC,kBAA6C;oBAC/DP,SAASQ,GAAG,CAACL,MAAMC;;;oBAErB;;wBAAOA;;;;IACT;;AAIO,SAAevB,iBAAiBsB,IAAgB;;;;;oBACrDH,SAASQ,GAAG,CAACL,MAAM;wBAAEI,OAAO;oBAAK;oBACjC;;wBAAME,IAAAA,iBAAO,EAACN,MAAMJ,oBAAoB;;;oBAAxC;;;;;;IACF;;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASW,WAAWlB,KAAa;IAC/B,IAAMmB,QAAkB,EAAE;IAC1B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAwB,EAAE;IAChC,IAAMC,aAAuB,EAAE;IAC/B,IAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,IAAMvB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACuB,IAAIC;QACtD,IAAMC,SAASD,MAAMpB,IAAI;QACzB,IAAMsB,eAAeC,IAAAA,uBAAY,EAACF;QAClC,IAAIC,aAAaE,MAAM,KAAK,GAAG;YAC7BN,cAAc;YACd,OAAO;QACT;QACAD,QAAQQ,IAAI,CAACH;YACR,kCAAA,2BAAA;;YAAL,QAAK,YAAeA,iCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;gBAA7B,IAAMvB,QAAN;gBACH,IAAIA,MAAM2B,QAAQ,EAAEV,WAAWS,IAAI,CAAC1B,MAAMV,IAAI;qBACzC;oBACHwB,MAAMY,IAAI,CAAC1B,MAAMV,IAAI;oBACrB0B,YAAYU,IAAI,CAAC1B,MAAMV,IAAI;gBAC7B;YACF;;YANK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAOL,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaM,eAAegC,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACK,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAM7B,QAAN;oBACH,IAAIA,MAAM2B,QAAQ,EAAEV,WAAWS,IAAI,CAAC1B,MAAMV,IAAI;yBACzC;wBACHwB,MAAMY,IAAI,CAAC1B,MAAMV,IAAI;wBACrByB,UAAUW,IAAI,CAAC1B,MAAMV,IAAI;oBAC3B;gBACF;;gBANK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAOP;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEwB,OAAAA;QAAOC,WAAAA;QAAWC,aAAAA;QAAaC,YAAAA;QAAYC,SAAAA;QAASC,aAAAA;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,SAAeW,eAAexB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMG,KAAK,EAAE;;;oBAClB;;wBAAMJ,KAAKyB,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAAC1B,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKyB,IAAI,CAAC;;;4CAAhB;4CACA,0FAA0F;4CAC1F,gFAAgF;4CAChF;;gDAAMnB,IAAAA,iBAAO,EAACN,MAAMJ,oBAAoB;;;4CAAxC;;;;;;4BACF;;;;oBAPA;oBAQAK,MAAMG,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASuB,kBAAkBnB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIvB,MAAMW,MAAM,GAAG,GAAG;QACpB,IAAMa,YAAYxB,MAAMyB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YAC3C,kCAAA,2BAAA;;YAAL,QAAK,YAAevD,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMwD,QAAN;gBACHR,WAAWR,IAAI,CAAC,AAAC,GAAgGgB,OAA9FvD,YAAY,CAACuD,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHN,YAAYV,IAAI,CAACY;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAIL,IAAIvB,UAAUU,MAAM,GAAG,GAAG;YACxB,IAAMkB,aAAa;YACnBR,UAAUT,IAAI,CAACiB;YACfN,WAAWX,IAAI,CAACX,UAAUwB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;QACpD;QACA,IAAIzB,YAAYS,MAAM,GAAG,GAAG;gBAK1BY;YAJA,IAAMM,cAAa;YACnB,IAAMC,WAAW5B,YAAYuB,GAAG,CAAC,SAACM;uBAAU,AAAC,IAA6F,OAA1F3D,OAAOqD,GAAG,CAAC,SAACG;2BAAU,AAAC,wCAA6C,OAANA,OAAM;mBAASD,IAAI,CAAC,SAAQ;eAAIA,IAAI,CAAC;YACnJN,UAAUT,IAAI,CAAC,AAAC,IAAqBkB,OAAlBD,aAAW,SAAgB,OAATC,UAAS;YAC9CP,WAAWX,IAAI,CAACV,YAAYuB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YACpDJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGrB,YAAY8B,OAAO,CAAC,SAACC;uBAAS7D,OAAOqD,GAAG,CAAC;2BAAMC,IAAAA,wBAAa,EAACO;;;QAClF;IACF;QAEK,mCAAA,4BAAA;;;YAAA,IAAMC,MAAN;gBAOHX;YANA,IAAMY,SAASD,IAAIE,WAAW;gBACzB,kCAAA,2BAAA;;gBAAL,QAAK,YAAehE,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;oBAAvB,IAAMwD,QAAN;oBACHR,WAAWR,IAAI,CAAC,AAAC,GAAuEgB,OAArEvD,YAAY,CAACuD,MAAM,EAAC,mDAAuD,OAANA,OAAM;oBAC9FN,YAAYV,IAAI,CAACc,IAAAA,wBAAa,EAACS;gBACjC;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILd,UAAUT,IAAI,CAAC,AAAC,IAA6F,OAA1FxC,OAAOqD,GAAG,CAAC,SAACG;uBAAU,AAAC,wCAA6C,OAANA,OAAM;eAASD,IAAI,CAAC,SAAQ;YAC7GJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGnD,OAAOqD,GAAG,CAAC;uBAAMC,IAAAA,wBAAa,EAACS;;QACpD;QARA,QAAK,aAAahC,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;QAAA;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEkC,UAAUjB,WAAWO,IAAI,CAAC;QAAQW,SAASjB,UAAUM,IAAI,CAAC,YAAY;QAAQY,QAAQ,AAAC,qBAAGjB,oBAAa,qBAAGC;IAAY;AACjI;AAIO,SAAepD,aAAaqB,IAAgB,EAAEX,KAAa,EAAE2D,IAAyB,EAAE/C,KAAoB;;YAsB7FgD,OArBdC,aAIEC,WAAWC,WAAWC,WAAWC,OACmC/C,aAApEC,OAAOC,WAAWC,aAAaC,YAAYC,SAASC,aAKtBc,oBAA9BkB,UAAUC,SAASC,QACrBQ,cACAC,UACAC,KAOAR,MACAS,MACAC;;;;oBAtBAT,cAAc9D,oBAAoBC;oBACxC,IAAI6D,gBAAgB,MAAM;wBACxB,MAAM,IAAIU,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEV,OAAvBA,YAAYhE,KAAK,EAAC,OAAqDG,OAAhD6D,YAAYxD,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQwD,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACmC/C,cAAAA,WAAWlB,QAA/EmB,QAAoED,YAApEC,OAAOC,YAA6DF,YAA7DE,WAAWC,cAAkDH,YAAlDG,aAAaC,aAAqCJ,YAArCI,YAAYC,UAAyBL,YAAzBK,SAASC,cAAgBN,YAAhBM;oBAC5D,IAAIA,eAAgBL,MAAMW,MAAM,KAAK,KAAKR,WAAWQ,MAAM,KAAK,GAAI;;;;yBAEhEX,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMgB,eAAexB,MAAMC;;;oBAA3B;;;oBAEgB0B,qBAAAA,kBAAkBnB,OAAOC,WAAWC,aAAaC,aAA/EkC,WAA8BlB,mBAA9BkB,UAAUC,UAAoBnB,mBAApBmB,SAASC,SAAWpB,mBAAXoB;oBACrBQ,eAAe3C,QAAQO,MAAM,GAAG,IAAI,mDAAmD;oBACvFqC,WAAW5C,QAAQO,MAAM,GAAG,IAAI,KAAK;oBACrCsC,MAAM,AAAC,qBACuBZ,OADHjC,QAAQO,MAAM,GAAG,IAAI,2BAA2B,IAAG,iDAC3BoC,OAArBV,UAAS,cAEzCM,OAFqDI,cAAa,4BAG5DT,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KACoBI,OADjBH,WAAU,wCACgB,OAATG;oBACrB;;wBAAMxD,KAAK6D,OAAO,CAACJ;;;oBAA1BR,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKa,GAAG,OAARb,OAAAA,AAAS,qBAAGF,eAAQ,qBAAInC,QAAQO,MAAM,GAAG;4BAAUmC;;;;oBAAjEI,OAAQ;oBACRC,UAAU/C,QAAQO,MAAM,KAAK,IAAIuC,OAAOA,KAAKK,MAAM,CAAC,SAACC;+BAAQpD,QAAQqD,KAAK,CAAC,SAACjD;mCAAWpC,OAAOsF,IAAI,CAAC,SAAC9B;oCAA8B4B;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAAC5B,MAAM,cAAV4B,wBAAAA,aAAc,IAAIhD;;;;oBAC1J;;wBAAO2C,QAAQS,KAAK,CAAC,GAAGd,OAAOrB,GAAG,CAAC;gCAAGoC,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAC3D;;AAIO,SAAS5F,mBAAmBuB,IAAgB;IACjD,OAAO;QACLsE,OAAO,SAAPA,MAAcjF,OAAO2D;;;;;;;gCAAsBhD;gCAAMX;gCAAO2D;;4BAAM;;gCAAMjD,SAASC;;;;;gCAA/CrB;oCAAgC;;;;;;;QAC9D4F,WAAW,SAAXA;mBAAiB7F,iBAAiBsB;;IACpC;AACF"}

package/dist/cjs/store/duckdb/reconcile.js

@@ -190,8 +190,13 @@ function reconcileContent(conn, touched, docs, _delta) {
                 case 3:
                     _state.sent();
                     // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
-                    // not here (lexical.ts's FtsIndexState).
-                    (0, _lexicalts.markContentStale)(conn);
+                    // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.
+                    return [
+                        4,
+                        (0, _lexicalts.markContentStale)(conn)
+                    ];
+                case 4:
+                    _state.sent();
                     return [
                         2
                     ];

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","appendRows","markContentStale","addColumns","names","exec","name","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAiDaA;;;eAAAA;;;wBAjDc;wBAGY;yBAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAE9F,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQb,GAAG,CAAC,SAACkB;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMC,IAAAA,oBAAU,EAACP,MAAM,WAAWd,iBAAiBC,oBAAoBe,KAAKd,GAAG,CAACG;;;oBAAhF;oBACA,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMtB,GAAG,CAAC,SAACwB;mCAAS,AAAC,sCAAsD,OAAjBvB,IAAAA,oBAAU,EAACuB,OAAM;2BAAWtB,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C4B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjB,yBAAyB;YACnC,MAAM,IAAIkB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElB,OAAtDiB,OAAM,kDAAwE,OAAxBjB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","appendRows","markContentStale","addColumns","names","exec","name","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAiDaA;;;eAAAA;;;wBAjDc;wBAGY;yBAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAE9F,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQb,GAAG,CAAC,SAACkB;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMC,IAAAA,oBAAU,EAACP,MAAM,WAAWd,iBAAiBC,oBAAoBe,KAAKd,GAAG,CAACG;;;oBAAhF;oBACA,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMiB,IAAAA,2BAAgB,EAACR;;;oBAAvB;;;;;;IACF;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMtB,GAAG,CAAC,SAACwB;mCAAS,AAAC,sCAAsD,OAAjBvB,IAAAA,oBAAU,EAACuB,OAAM;2BAAWtB,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C4B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjB,yBAAyB;YACnC,MAAM,IAAIkB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElB,OAAtDiB,OAAM,kDAAwE,OAAxBjB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}

package/dist/cjs/store/duckdb/store.js

@@ -263,10 +263,8 @@ function _wrap_async_generator(fn) {
     };
 }
 var CAPABILITIES = new Set([
-    'lexical',
-    'phrases',
     'vectors',
-    'sql-functions'
+    'segment'
 ]);
 function createStore(instance, duckdb, conn) {
     var lex = (0, _lexicalts.createLexicalIndex)(conn);

package/dist/cjs/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACb;4BACF;yBACQ;0BAC2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB;AAIxG,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,IAAMC,MAAMC,IAAAA,6BAAkB,EAACF;IAC/B,OAAO;QACLG,MAAM;QACNC,cAAcT;QACRU,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACf;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAgB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAAChB,MAAMc,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOlB,IAAIkB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACtB;;YAC3BuB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,QAAQC,MAAM0B,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,QAAQ6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAACrC,QAAQC,MAAM0B,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAACtC,MAAMkC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGjC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBmC;;;;gCAAW;;oCAAM1C,OAAOQ,OAAO,CAACD;;;gCAAhCmC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE1C,MAAMsC,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjF5D,OAAO6D,cAAc;oBACrB9D,SAAS+D,SAAS;;;;;YACpB;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. segment() registers as a UDF (sql-functions.ts).\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors', 'segment']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACb;4BACF;yBACQ;0BAC2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;CAAU;AAI5E,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,IAAMC,MAAMC,IAAAA,6BAAkB,EAACF;IAC/B,OAAO;QACLG,MAAM;QACNC,cAAcT;QACRU,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACf;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAgB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAAChB,MAAMc,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOlB,IAAIkB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACtB;;YAC3BuB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,QAAQC,MAAM0B,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,QAAQ6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAACrC,QAAQC,MAAM0B,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAACtC,MAAMkC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGjC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBmC;;;;gCAAW;;oCAAM1C,OAAOQ,OAAO,CAACD;;;gCAAhCmC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE1C,MAAMsC,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjF5D,OAAO6D,cAAc;oBACrB9D,SAAS+D,SAAS;;;;;YACpB;;IACF;AACF"}