sensemaking 0.24.4 → 0.24.6

package/dist/cjs/store/duckdb/lexical.js

@@ -25,6 +25,7 @@ var _errorsts = require("../../errors.js");
 var _segmentts = require("../../text/segment.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
+var _orderedbm25ts = require("./ordered-bm25.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -190,7 +191,7 @@ var FIELD_WEIGHT = {
     text: 1
 };
 // FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently
-// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.
+// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.
 var FTS5_OPERATORS = [
     {
         label: 'prefix query',
@@ -211,10 +212,14 @@ var FTS5_OPERATORS = [
     {
         label: 'column filter',
         re: RegExp("(?:^|\\s)[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376-\\u0377\\u037A-\\u037D\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u052F\\u0531-\\u0556\\u0559\\u0560-\\u0588\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u064A\\u066E-\\u066F\\u0671-\\u06D3\\u06D5\\u06E5-\\u06E6\\u06EE-\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4-\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u0860-\\u086A\\u0870-\\u0887\\u0889-\\u088F\\u08A0-\\u08C9\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098C\\u098F-\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC-\\u09DD\\u09DF-\\u09E1\\u09F0-\\u09F1\\u09FC\\u0A05-\\u0A0A\\u0A0F-\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32-\\u0A33\\u0A35-\\u0A36\\u0A38-\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2-\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0-\\u0AE1\\u0AF9\\u0B05-\\u0B0C\\u0B0F-\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32-\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C-\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99-\\u0B9A\\u0B9C\\u0B9E-\\u0B9F\\u0BA3-\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C39\\u0C3D\\u0C58-\\u0C5A\\u0C5C-\\u0C5D\\u0C60-\\u0C61\\u0C80\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDC-\\u0CDE\\u0CE0-\\u0CE1\\u0CF1-\\u0CF2\\u0D04-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D54-\\u0D56\\u0D5F-\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32-\\u0E33\\u0E40-\\u0E46\\u0E81-\\u0E82\\u0E84\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB0\\u0EB2-\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065-\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F5\\u13F8-\\u13FD\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16F1-\\u16F8\\u1700-\\u1711\\u171F-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1878\\u1880-\\u1884\\u1887-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191E\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19B0-\\u19C9\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4C\\u1B83-\\u1BA0\\u1BAE-\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1C80-\\u1C8A\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1CE9-\\u1CEC\\u1CEE-\\u1CF3\\u1CF5-\\u1CF6\\u1CFA\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183-\\u2184\\u2C00-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2-\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3006\\u3031-\\u3035\\u303B-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312F\\u3131-\\u318E\\u31A0-\\u31BF\\u31F0-\\u31FF\\u3400-\\u4DBF\\u4E00-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A-\\uA62B\\uA640-\\uA66E\\uA67F-\\uA69D\\uA6A0-\\uA6E5\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA7DC\\uA7F1-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA8FD-\\uA8FE\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uA9E0-\\uA9E4\\uA9E6-\\uA9EF\\uA9FA-\\uA9FE\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA7E-\\uAAAF\\uAAB1\\uAAB5-\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB30-\\uAB5A\\uAB5C-\\uAB69\\uAB70-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40-\\uFB41\\uFB43-\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC\\u{10000}-\\u{1000B}\\u{1000D}-\\u{10026}\\u{10028}-\\u{1003A}\\u{1003C}-\\u{1003D}\\u{1003F}-\\u{1004D}\\u{10050}-\\u{1005D}\\u{10080}-\\u{100FA}\\u{10280}-\\u{1029C}\\u{102A0}-\\u{102D0}\\u{10300}-\\u{1031F}\\u{1032D}-\\u{10340}\\u{10342}-\\u{10349}\\u{10350}-\\u{10375}\\u{10380}-\\u{1039D}\\u{103A0}-\\u{103C3}\\u{103C8}-\\u{103CF}\\u{10400}-\\u{1049D}\\u{104B0}-\\u{104D3}\\u{104D8}-\\u{104FB}\\u{10500}-\\u{10527}\\u{10530}-\\u{10563}\\u{10570}-\\u{1057A}\\u{1057C}-\\u{1058A}\\u{1058C}-\\u{10592}\\u{10594}-\\u{10595}\\u{10597}-\\u{105A1}\\u{105A3}-\\u{105B1}\\u{105B3}-\\u{105B9}\\u{105BB}-\\u{105BC}\\u{105C0}-\\u{105F3}\\u{10600}-\\u{10736}\\u{10740}-\\u{10755}\\u{10760}-\\u{10767}\\u{10780}-\\u{10785}\\u{10787}-\\u{107B0}\\u{107B2}-\\u{107BA}\\u{10800}-\\u{10805}\\u{10808}\\u{1080A}-\\u{10835}\\u{10837}-\\u{10838}\\u{1083C}\\u{1083F}-\\u{10855}\\u{10860}-\\u{10876}\\u{10880}-\\u{1089E}\\u{108E0}-\\u{108F2}\\u{108F4}-\\u{108F5}\\u{10900}-\\u{10915}\\u{10920}-\\u{10939}\\u{10940}-\\u{10959}\\u{10980}-\\u{109B7}\\u{109BE}-\\u{109BF}\\u{10A00}\\u{10A10}-\\u{10A13}\\u{10A15}-\\u{10A17}\\u{10A19}-\\u{10A35}\\u{10A60}-\\u{10A7C}\\u{10A80}-\\u{10A9C}\\u{10AC0}-\\u{10AC7}\\u{10AC9}-\\u{10AE4}\\u{10B00}-\\u{10B35}\\u{10B40}-\\u{10B55}\\u{10B60}-\\u{10B72}\\u{10B80}-\\u{10B91}\\u{10C00}-\\u{10C48}\\u{10C80}-\\u{10CB2}\\u{10CC0}-\\u{10CF2}\\u{10D00}-\\u{10D23}\\u{10D4A}-\\u{10D65}\\u{10D6F}-\\u{10D85}\\u{10E80}-\\u{10EA9}\\u{10EB0}-\\u{10EB1}\\u{10EC2}-\\u{10EC7}\\u{10F00}-\\u{10F1C}\\u{10F27}\\u{10F30}-\\u{10F45}\\u{10F70}-\\u{10F81}\\u{10FB0}-\\u{10FC4}\\u{10FE0}-\\u{10FF6}\\u{11003}-\\u{11037}\\u{11071}-\\u{11072}\\u{11075}\\u{11083}-\\u{110AF}\\u{110D0}-\\u{110E8}\\u{11103}-\\u{11126}\\u{11144}\\u{11147}\\u{11150}-\\u{11172}\\u{11176}\\u{11183}-\\u{111B2}\\u{111C1}-\\u{111C4}\\u{111DA}\\u{111DC}\\u{11200}-\\u{11211}\\u{11213}-\\u{1122B}\\u{1123F}-\\u{11240}\\u{11280}-\\u{11286}\\u{11288}\\u{1128A}-\\u{1128D}\\u{1128F}-\\u{1129D}\\u{1129F}-\\u{112A8}\\u{112B0}-\\u{112DE}\\u{11305}-\\u{1130C}\\u{1130F}-\\u{11310}\\u{11313}-\\u{11328}\\u{1132A}-\\u{11330}\\u{11332}-\\u{11333}\\u{11335}-\\u{11339}\\u{1133D}\\u{11350}\\u{1135D}-\\u{11361}\\u{11380}-\\u{11389}\\u{1138B}\\u{1138E}\\u{11390}-\\u{113B5}\\u{113B7}\\u{113D1}\\u{113D3}\\u{11400}-\\u{11434}\\u{11447}-\\u{1144A}\\u{1145F}-\\u{11461}\\u{11480}-\\u{114AF}\\u{114C4}-\\u{114C5}\\u{114C7}\\u{11580}-\\u{115AE}\\u{115D8}-\\u{115DB}\\u{11600}-\\u{1162F}\\u{11644}\\u{11680}-\\u{116AA}\\u{116B8}\\u{11700}-\\u{1171A}\\u{11740}-\\u{11746}\\u{11800}-\\u{1182B}\\u{118A0}-\\u{118DF}\\u{118FF}-\\u{11906}\\u{11909}\\u{1190C}-\\u{11913}\\u{11915}-\\u{11916}\\u{11918}-\\u{1192F}\\u{1193F}\\u{11941}\\u{119A0}-\\u{119A7}\\u{119AA}-\\u{119D0}\\u{119E1}\\u{119E3}\\u{11A00}\\u{11A0B}-\\u{11A32}\\u{11A3A}\\u{11A50}\\u{11A5C}-\\u{11A89}\\u{11A9D}\\u{11AB0}-\\u{11AF8}\\u{11BC0}-\\u{11BE0}\\u{11C00}-\\u{11C08}\\u{11C0A}-\\u{11C2E}\\u{11C40}\\u{11C72}-\\u{11C8F}\\u{11D00}-\\u{11D06}\\u{11D08}-\\u{11D09}\\u{11D0B}-\\u{11D30}\\u{11D46}\\u{11D60}-\\u{11D65}\\u{11D67}-\\u{11D68}\\u{11D6A}-\\u{11D89}\\u{11D98}\\u{11DB0}-\\u{11DDB}\\u{11EE0}-\\u{11EF2}\\u{11F02}\\u{11F04}-\\u{11F10}\\u{11F12}-\\u{11F33}\\u{11FB0}\\u{12000}-\\u{12399}\\u{12480}-\\u{12543}\\u{12F90}-\\u{12FF0}\\u{13000}-\\u{1342F}\\u{13441}-\\u{13446}\\u{13460}-\\u{143FA}\\u{14400}-\\u{14646}\\u{16100}-\\u{1611D}\\u{16800}-\\u{16A38}\\u{16A40}-\\u{16A5E}\\u{16A70}-\\u{16ABE}\\u{16AD0}-\\u{16AED}\\u{16B00}-\\u{16B2F}\\u{16B40}-\\u{16B43}\\u{16B63}-\\u{16B77}\\u{16B7D}-\\u{16B8F}\\u{16D40}-\\u{16D6C}\\u{16E40}-\\u{16E7F}\\u{16EA0}-\\u{16EB8}\\u{16EBB}-\\u{16ED3}\\u{16F00}-\\u{16F4A}\\u{16F50}\\u{16F93}-\\u{16F9F}\\u{16FE0}-\\u{16FE1}\\u{16FE3}\\u{16FF2}-\\u{16FF3}\\u{17000}-\\u{18CD5}\\u{18CFF}-\\u{18D1E}\\u{18D80}-\\u{18DF2}\\u{1AFF0}-\\u{1AFF3}\\u{1AFF5}-\\u{1AFFB}\\u{1AFFD}-\\u{1AFFE}\\u{1B000}-\\u{1B122}\\u{1B132}\\u{1B150}-\\u{1B152}\\u{1B155}\\u{1B164}-\\u{1B167}\\u{1B170}-\\u{1B2FB}\\u{1BC00}-\\u{1BC6A}\\u{1BC70}-\\u{1BC7C}\\u{1BC80}-\\u{1BC88}\\u{1BC90}-\\u{1BC99}\\u{1D400}-\\u{1D454}\\u{1D456}-\\u{1D49C}\\u{1D49E}-\\u{1D49F}\\u{1D4A2}\\u{1D4A5}-\\u{1D4A6}\\u{1D4A9}-\\u{1D4AC}\\u{1D4AE}-\\u{1D4B9}\\u{1D4BB}\\u{1D4BD}-\\u{1D4C3}\\u{1D4C5}-\\u{1D505}\\u{1D507}-\\u{1D50A}\\u{1D50D}-\\u{1D514}\\u{1D516}-\\u{1D51C}\\u{1D51E}-\\u{1D539}\\u{1D53B}-\\u{1D53E}\\u{1D540}-\\u{1D544}\\u{1D546}\\u{1D54A}-\\u{1D550}\\u{1D552}-\\u{1D6A5}\\u{1D6A8}-\\u{1D6C0}\\u{1D6C2}-\\u{1D6DA}\\u{1D6DC}-\\u{1D6FA}\\u{1D6FC}-\\u{1D714}\\u{1D716}-\\u{1D734}\\u{1D736}-\\u{1D74E}\\u{1D750}-\\u{1D76E}\\u{1D770}-\\u{1D788}\\u{1D78A}-\\u{1D7A8}\\u{1D7AA}-\\u{1D7C2}\\u{1D7C4}-\\u{1D7CB}\\u{1DF00}-\\u{1DF1E}\\u{1DF25}-\\u{1DF2A}\\u{1E030}-\\u{1E06D}\\u{1E100}-\\u{1E12C}\\u{1E137}-\\u{1E13D}\\u{1E14E}\\u{1E290}-\\u{1E2AD}\\u{1E2C0}-\\u{1E2EB}\\u{1E4D0}-\\u{1E4EB}\\u{1E5D0}-\\u{1E5ED}\\u{1E5F0}\\u{1E6C0}-\\u{1E6DE}\\u{1E6E0}-\\u{1E6E2}\\u{1E6E4}-\\u{1E6E5}\\u{1E6E7}-\\u{1E6ED}\\u{1E6F0}-\\u{1E6F4}\\u{1E6FE}-\\u{1E6FF}\\u{1E7E0}-\\u{1E7E6}\\u{1E7E8}-\\u{1E7EB}\\u{1E7ED}-\\u{1E7EE}\\u{1E7F0}-\\u{1E7FE}\\u{1E800}-\\u{1E8C4}\\u{1E900}-\\u{1E943}\\u{1E94B}\\u{1EE00}-\\u{1EE03}\\u{1EE05}-\\u{1EE1F}\\u{1EE21}-\\u{1EE22}\\u{1EE24}\\u{1EE27}\\u{1EE29}-\\u{1EE32}\\u{1EE34}-\\u{1EE37}\\u{1EE39}\\u{1EE3B}\\u{1EE42}\\u{1EE47}\\u{1EE49}\\u{1EE4B}\\u{1EE4D}-\\u{1EE4F}\\u{1EE51}-\\u{1EE52}\\u{1EE54}\\u{1EE57}\\u{1EE59}\\u{1EE5B}\\u{1EE5D}\\u{1EE5F}\\u{1EE61}-\\u{1EE62}\\u{1EE64}\\u{1EE67}-\\u{1EE6A}\\u{1EE6C}-\\u{1EE72}\\u{1EE74}-\\u{1EE77}\\u{1EE79}-\\u{1EE7C}\\u{1EE7E}\\u{1EE80}-\\u{1EE89}\\u{1EE8B}-\\u{1EE9B}\\u{1EEA1}-\\u{1EEA3}\\u{1EEA5}-\\u{1EEA9}\\u{1EEAB}-\\u{1EEBB}\\u{20000}-\\u{2A6DF}\\u{2A700}-\\u{2B81D}\\u{2B820}-\\u{2CEAD}\\u{2CEB0}-\\u{2EBE0}\\u{2EBF0}-\\u{2EE5D}\\u{2F800}-\\u{2FA1D}\\u{30000}-\\u{3134A}\\u{31350}-\\u{33479}_]\\w*\\s*:", "u")
+    },
+    {
+        label: 'grouping operator',
+        re: /[()]/
     }
 ];
 function unsupportedOperator(terms) {
-    var withoutPhrases = terms.replace(/"[^"]*"/g, ' ');
+    var withoutPhrases = terms.replace(/"[^"]*"/g, 'phrase');
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
     try {
         for(var _iterator = FTS5_OPERATORS[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
@@ -266,7 +271,7 @@ function stateFor(conn) {
                 case 1:
                     // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that
                     // never built an index, or one a prior process marked stale and never got to clear.
-                    state = (_tmp.stale = _state.sent() !== '0', _tmp);
+                    state = (_tmp.stale = _state.sent() !== '0', _tmp.orderedMacroReady = false, _tmp);
                     ftsState.set(conn, state);
                     _state.label = 2;
                 case 2:
@@ -284,7 +289,8 @@ function markContentStale(conn) {
             switch(_state.label){
                 case 0:
                     ftsState.set(conn, {
-                        stale: true
+                        stale: true,
+                        orderedMacroReady: false
                     });
                     return [
                         4,
@@ -405,6 +411,7 @@ function ensureFtsFresh(conn, state) {
                     if (!state.stale) return [
                         2
                     ];
+                    state.orderedMacroReady = false;
                     return [
                         4,
                         conn.exec('INSTALL fts; LOAD fts;')
@@ -465,7 +472,7 @@ function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
         try {
             for(var _iterator = FIELDS[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
                 var field = _step.value;
-                scoreParts.push("".concat(FIELD_WEIGHT[field], '.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := \'').concat(field, "', conjunctive := false), 0)"));
+                scoreParts.push("".concat(FIELD_WEIGHT[field], ".0 * COALESCE(").concat(_orderedbm25ts.ORDERED_BM25_MACRO, '(content."path", ?, fields := \'').concat(field, "', conjunctive := false), 0)"));
                 scoreParams.push(wordQuery);
             }
         } catch (err) {
@@ -483,13 +490,13 @@ function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
             }
         }
         if (bareWords.length > 0) {
-            var nativeGate = 'fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL';
+            var nativeGate = "".concat(_orderedbm25ts.ORDERED_BM25_MACRO, '(content."path", ?, conjunctive := true) IS NOT NULL');
             gateParts.push(nativeGate);
             gateParams.push(bareWords.map(_segmentts.foldForSearch).join(' '));
         }
         if (phraseWords.length > 0) {
             var _gateParams;
-            var nativeGate1 = 'fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL';
+            var nativeGate1 = "".concat(_orderedbm25ts.ORDERED_BM25_MACRO, '(content."path", ?, conjunctive := true) IS NOT NULL');
             var fallback = phraseWords.map(function(_word) {
                 return "(".concat(FIELDS.map(function(field) {
                     return "contains(strip_accents(lower(content.".concat(field, ")), ?)");
@@ -587,6 +594,18 @@ function queryLexical(conn, terms, opts, state) {
                     _state.sent();
                     _state.label = 2;
                 case 2:
+                    if (!(words.length > 0)) return [
+                        3,
+                        4
+                    ];
+                    return [
+                        4,
+                        (0, _orderedbm25ts.ensureOrderedBm25)(conn, state)
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
                     _buildScoreAndGate = buildScoreAndGate(words, bareWords, phraseWords, substrings), scoreSql = _buildScoreAndGate.scoreSql, gateSql = _buildScoreAndGate.gateSql, params = _buildScoreAndGate.params;
                     selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
                     sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';
@@ -595,7 +614,7 @@ function queryLexical(conn, terms, opts, state) {
                         4,
                         conn.prepare(sql)
                     ];
-                case 3:
+                case 5:
                     stmt = _state.sent();
                     return [
                         4,
@@ -603,7 +622,7 @@ function queryLexical(conn, terms, opts, state) {
                             limit
                         ])))
                     ];
-                case 4:
+                case 6:
                     rows = _state.sent();
                     matches = phrases.length === 0 ? rows : rows.filter(function(row) {
                         return phrases.every(function(phrase) {

package/dist/cjs/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0' };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","getMeta","stale","set","setMeta","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","searchTokens","length","push","unspaced","split","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","foldForSearch","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","matches","SenseError","prepare","all","filter","row","every","some","matchesSearchPhrase","slice","path","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QA4L9CA;eAAAA;;QApIMC;eAAAA;;QAuGAC;eAAAA;;;wBA9JK;yBACwD;wBAClD;6BACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,kGAAkG;AAClG,kGAAkG;AAClG,IAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAAeC,SAASC,IAAgB;;YAClCC;;;;oBAAAA,QAAQJ,SAASK,GAAG,CAACF;yBACrB,CAACC,OAAD;;;;;oBAGgB;;wBAAME,IAAAA,iBAAO,EAACH,MAAMJ;;;oBAFtC,2FAA2F;oBAC3F,oFAAoF;oBACpFK,cAAUG,QAAO,AAAC,kBAA6C;oBAC/DP,SAASQ,GAAG,CAACL,MAAMC;;;oBAErB;;wBAAOA;;;;IACT;;AAIO,SAAevB,iBAAiBsB,IAAgB;;;;;oBACrDH,SAASQ,GAAG,CAACL,MAAM;wBAAEI,OAAO;oBAAK;oBACjC;;wBAAME,IAAAA,iBAAO,EAACN,MAAMJ,oBAAoB;;;oBAAxC;;;;;;IACF;;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASW,WAAWlB,KAAa;IAC/B,IAAMmB,QAAkB,EAAE;IAC1B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAwB,EAAE;IAChC,IAAMC,aAAuB,EAAE;IAC/B,IAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,IAAMvB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACuB,IAAIC;QACtD,IAAMC,SAASD,MAAMpB,IAAI;QACzB,IAAMsB,eAAeC,IAAAA,uBAAY,EAACF;QAClC,IAAIC,aAAaE,MAAM,KAAK,GAAG;YAC7BN,cAAc;YACd,OAAO;QACT;QACAD,QAAQQ,IAAI,CAACH;YACR,kCAAA,2BAAA;;YAAL,QAAK,YAAeA,iCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;gBAA7B,IAAMvB,QAAN;gBACH,IAAIA,MAAM2B,QAAQ,EAAEV,WAAWS,IAAI,CAAC1B,MAAMV,IAAI;qBACzC;oBACHwB,MAAMY,IAAI,CAAC1B,MAAMV,IAAI;oBACrB0B,YAAYU,IAAI,CAAC1B,MAAMV,IAAI;gBAC7B;YACF;;YANK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAOL,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaM,eAAegC,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACK,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAM7B,QAAN;oBACH,IAAIA,MAAM2B,QAAQ,EAAEV,WAAWS,IAAI,CAAC1B,MAAMV,IAAI;yBACzC;wBACHwB,MAAMY,IAAI,CAAC1B,MAAMV,IAAI;wBACrByB,UAAUW,IAAI,CAAC1B,MAAMV,IAAI;oBAC3B;gBACF;;gBANK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAOP;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEwB,OAAAA;QAAOC,WAAAA;QAAWC,aAAAA;QAAaC,YAAAA;QAAYC,SAAAA;QAASC,aAAAA;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,SAAeW,eAAexB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMG,KAAK,EAAE;;;oBAClB;;wBAAMJ,KAAKyB,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAAC1B,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKyB,IAAI,CAAC;;;4CAAhB;4CACA,0FAA0F;4CAC1F,gFAAgF;4CAChF;;gDAAMnB,IAAAA,iBAAO,EAACN,MAAMJ,oBAAoB;;;4CAAxC;;;;;;4BACF;;;;oBAPA;oBAQAK,MAAMG,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASuB,kBAAkBnB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIvB,MAAMW,MAAM,GAAG,GAAG;QACpB,IAAMa,YAAYxB,MAAMyB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YAC3C,kCAAA,2BAAA;;YAAL,QAAK,YAAevD,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMwD,QAAN;gBACHR,WAAWR,IAAI,CAAC,AAAC,GAAgGgB,OAA9FvD,YAAY,CAACuD,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHN,YAAYV,IAAI,CAACY;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAIL,IAAIvB,UAAUU,MAAM,GAAG,GAAG;YACxB,IAAMkB,aAAa;YACnBR,UAAUT,IAAI,CAACiB;YACfN,WAAWX,IAAI,CAACX,UAAUwB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;QACpD;QACA,IAAIzB,YAAYS,MAAM,GAAG,GAAG;gBAK1BY;YAJA,IAAMM,cAAa;YACnB,IAAMC,WAAW5B,YAAYuB,GAAG,CAAC,SAACM;uBAAU,AAAC,IAA6F,OAA1F3D,OAAOqD,GAAG,CAAC,SAACG;2BAAU,AAAC,wCAA6C,OAANA,OAAM;mBAASD,IAAI,CAAC,SAAQ;eAAIA,IAAI,CAAC;YACnJN,UAAUT,IAAI,CAAC,AAAC,IAAqBkB,OAAlBD,aAAW,SAAgB,OAATC,UAAS;YAC9CP,WAAWX,IAAI,CAACV,YAAYuB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YACpDJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGrB,YAAY8B,OAAO,CAAC,SAACC;uBAAS7D,OAAOqD,GAAG,CAAC;2BAAMC,IAAAA,wBAAa,EAACO;;;QAClF;IACF;QAEK,mCAAA,4BAAA;;;YAAA,IAAMC,MAAN;gBAOHX;YANA,IAAMY,SAASD,IAAIE,WAAW;gBACzB,kCAAA,2BAAA;;gBAAL,QAAK,YAAehE,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;oBAAvB,IAAMwD,QAAN;oBACHR,WAAWR,IAAI,CAAC,AAAC,GAAuEgB,OAArEvD,YAAY,CAACuD,MAAM,EAAC,mDAAuD,OAANA,OAAM;oBAC9FN,YAAYV,IAAI,CAACc,IAAAA,wBAAa,EAACS;gBACjC;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILd,UAAUT,IAAI,CAAC,AAAC,IAA6F,OAA1FxC,OAAOqD,GAAG,CAAC,SAACG;uBAAU,AAAC,wCAA6C,OAANA,OAAM;eAASD,IAAI,CAAC,SAAQ;YAC7GJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGnD,OAAOqD,GAAG,CAAC;uBAAMC,IAAAA,wBAAa,EAACS;;QACpD;QARA,QAAK,aAAahC,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;QAAA;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEkC,UAAUjB,WAAWO,IAAI,CAAC;QAAQW,SAASjB,UAAUM,IAAI,CAAC,YAAY;QAAQY,QAAQ,AAAC,qBAAGjB,oBAAa,qBAAGC;IAAY;AACjI;AAIO,SAAepD,aAAaqB,IAAgB,EAAEX,KAAa,EAAE2D,IAAyB,EAAE/C,KAAoB;;YAsB7FgD,OArBdC,aAIEC,WAAWC,WAAWC,WAAWC,OACmC/C,aAApEC,OAAOC,WAAWC,aAAaC,YAAYC,SAASC,aAKtBc,oBAA9BkB,UAAUC,SAASC,QACrBQ,cACAC,UACAC,KAOAR,MACAS,MACAC;;;;oBAtBAT,cAAc9D,oBAAoBC;oBACxC,IAAI6D,gBAAgB,MAAM;wBACxB,MAAM,IAAIU,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEV,OAAvBA,YAAYhE,KAAK,EAAC,OAAqDG,OAAhD6D,YAAYxD,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQwD,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACmC/C,cAAAA,WAAWlB,QAA/EmB,QAAoED,YAApEC,OAAOC,YAA6DF,YAA7DE,WAAWC,cAAkDH,YAAlDG,aAAaC,aAAqCJ,YAArCI,YAAYC,UAAyBL,YAAzBK,SAASC,cAAgBN,YAAhBM;oBAC5D,IAAIA,eAAgBL,MAAMW,MAAM,KAAK,KAAKR,WAAWQ,MAAM,KAAK,GAAI;;;;yBAEhEX,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMgB,eAAexB,MAAMC;;;oBAA3B;;;oBAEgB0B,qBAAAA,kBAAkBnB,OAAOC,WAAWC,aAAaC,aAA/EkC,WAA8BlB,mBAA9BkB,UAAUC,UAAoBnB,mBAApBmB,SAASC,SAAWpB,mBAAXoB;oBACrBQ,eAAe3C,QAAQO,MAAM,GAAG,IAAI,mDAAmD;oBACvFqC,WAAW5C,QAAQO,MAAM,GAAG,IAAI,KAAK;oBACrCsC,MAAM,AAAC,qBACuBZ,OADHjC,QAAQO,MAAM,GAAG,IAAI,2BAA2B,IAAG,iDAC3BoC,OAArBV,UAAS,cAEzCM,OAFqDI,cAAa,4BAG5DT,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KACoBI,OADjBH,WAAU,wCACgB,OAATG;oBACrB;;wBAAMxD,KAAK6D,OAAO,CAACJ;;;oBAA1BR,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKa,GAAG,OAARb,OAAAA,AAAS,qBAAGF,eAAQ,qBAAInC,QAAQO,MAAM,GAAG;4BAAUmC;;;;oBAAjEI,OAAQ;oBACRC,UAAU/C,QAAQO,MAAM,KAAK,IAAIuC,OAAOA,KAAKK,MAAM,CAAC,SAACC;+BAAQpD,QAAQqD,KAAK,CAAC,SAACjD;mCAAWpC,OAAOsF,IAAI,CAAC,SAAC9B;oCAA8B4B;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAAC5B,MAAM,cAAV4B,wBAAAA,aAAc,IAAIhD;;;;oBAC1J;;wBAAO2C,QAAQS,KAAK,CAAC,GAAGd,OAAOrB,GAAG,CAAC;gCAAGoC,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAC3D;;AAIO,SAAS5F,mBAAmBuB,IAAgB;IACjD,OAAO;QACLsE,OAAO,SAAPA,MAAcjF,OAAO2D;;;;;;;gCAAsBhD;gCAAMX;gCAAO2D;;4BAAM;;gCAAMjD,SAASC;;;;;gCAA/CrB;oCAAgC;;;;;;;QAC9D4F,WAAW,SAAXA;mBAAiB7F,iBAAiBsB;;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n  orderedMacroReady: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0', orderedMacroReady: false };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true, orderedMacroReady: false });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  state.orderedMacroReady = false;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n  if (words.length > 0) await ensureOrderedBm25(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","getMeta","stale","orderedMacroReady","set","setMeta","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","searchTokens","length","push","unspaced","split","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","foldForSearch","join","field","ORDERED_BM25_MACRO","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","matches","SenseError","ensureOrderedBm25","prepare","all","filter","row","every","some","matchesSearchPhrase","slice","path","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAiM9CA;eAAAA;;QAtIMC;eAAAA;;QAwGAC;eAAAA;;;wBAlKK;yBACwD;wBAClD;6BACD;6BAEsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtD,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAOA,kGAAkG;AAClG,kGAAkG;AAClG,IAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAAeC,SAASC,IAAgB;;YAClCC;;;;oBAAAA,QAAQJ,SAASK,GAAG,CAACF;yBACrB,CAACC,OAAD;;;;;oBAGgB;;wBAAME,IAAAA,iBAAO,EAACH,MAAMJ;;;oBAFtC,2FAA2F;oBAC3F,oFAAoF;oBACpFK,cAAUG,QAAO,AAAC,kBAA6C,UAAKC,oBAAmB;oBACvFR,SAASS,GAAG,CAACN,MAAMC;;;oBAErB;;wBAAOA;;;;IACT;;AAIO,SAAevB,iBAAiBsB,IAAgB;;;;;oBACrDH,SAASS,GAAG,CAACN,MAAM;wBAAEI,OAAO;wBAAMC,mBAAmB;oBAAM;oBAC3D;;wBAAME,IAAAA,iBAAO,EAACP,MAAMJ,oBAAoB;;;oBAAxC;;;;;;IACF;;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASY,WAAWnB,KAAa;IAC/B,IAAMoB,QAAkB,EAAE;IAC1B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAwB,EAAE;IAChC,IAAMC,aAAuB,EAAE;IAC/B,IAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,IAAMxB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACwB,IAAIC;QACtD,IAAMC,SAASD,MAAMrB,IAAI;QACzB,IAAMuB,eAAeC,IAAAA,uBAAY,EAACF;QAClC,IAAIC,aAAaE,MAAM,KAAK,GAAG;YAC7BN,cAAc;YACd,OAAO;QACT;QACAD,QAAQQ,IAAI,CAACH;YACR,kCAAA,2BAAA;;YAAL,QAAK,YAAeA,iCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;gBAA7B,IAAMxB,QAAN;gBACH,IAAIA,MAAM4B,QAAQ,EAAEV,WAAWS,IAAI,CAAC3B,MAAMV,IAAI;qBACzC;oBACHyB,MAAMY,IAAI,CAAC3B,MAAMV,IAAI;oBACrB2B,YAAYU,IAAI,CAAC3B,MAAMV,IAAI;gBAC7B;YACF;;YANK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAOL,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaM,eAAeiC,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACK,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAM9B,QAAN;oBACH,IAAIA,MAAM4B,QAAQ,EAAEV,WAAWS,IAAI,CAAC3B,MAAMV,IAAI;yBACzC;wBACHyB,MAAMY,IAAI,CAAC3B,MAAMV,IAAI;wBACrB0B,UAAUW,IAAI,CAAC3B,MAAMV,IAAI;oBAC3B;gBACF;;gBANK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAOP;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEyB,OAAAA;QAAOC,WAAAA;QAAWC,aAAAA;QAAaC,YAAAA;QAAYC,SAAAA;QAASC,aAAAA;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,SAAeW,eAAezB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMG,KAAK,EAAE;;;oBAClBH,MAAMI,iBAAiB,GAAG;oBAC1B;;wBAAML,KAAK0B,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAAC3B,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAK0B,IAAI,CAAC;;;4CAAhB;4CACA,0FAA0F;4CAC1F,gFAAgF;4CAChF;;gDAAMnB,IAAAA,iBAAO,EAACP,MAAMJ,oBAAoB;;;4CAAxC;;;;;;4BACF;;;;oBAPA;oBAQAK,MAAMG,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASwB,kBAAkBnB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIvB,MAAMW,MAAM,GAAG,GAAG;QACpB,IAAMa,YAAYxB,MAAMyB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YAC3C,kCAAA,2BAAA;;YAAL,QAAK,YAAexD,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMyD,QAAN;gBACHR,WAAWR,IAAI,CAAC,AAAC,GAAsCiB,OAApCzD,YAAY,CAACwD,MAAM,EAAC,kBAAoEA,OAApDC,iCAAkB,EAAC,oCAAuC,OAAND,OAAM;gBACjHN,YAAYV,IAAI,CAACY;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAIL,IAAIvB,UAAUU,MAAM,GAAG,GAAG;YACxB,IAAMmB,aAAa,AAAC,GAAqB,OAAnBD,iCAAkB,EAAC;YACzCR,UAAUT,IAAI,CAACkB;YACfP,WAAWX,IAAI,CAACX,UAAUwB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;QACpD;QACA,IAAIzB,YAAYS,MAAM,GAAG,GAAG;gBAK1BY;YAJA,IAAMO,cAAa,AAAC,GAAqB,OAAnBD,iCAAkB,EAAC;YACzC,IAAME,WAAW7B,YAAYuB,GAAG,CAAC,SAACO;uBAAU,AAAC,IAA6F,OAA1F7D,OAAOsD,GAAG,CAAC,SAACG;2BAAU,AAAC,wCAA6C,OAANA,OAAM;mBAASD,IAAI,CAAC,SAAQ;eAAIA,IAAI,CAAC;YACnJN,UAAUT,IAAI,CAAC,AAAC,IAAqBmB,OAAlBD,aAAW,SAAgB,OAATC,UAAS;YAC9CR,WAAWX,IAAI,CAACV,YAAYuB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YACpDJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGrB,YAAY+B,OAAO,CAAC,SAACC;uBAAS/D,OAAOsD,GAAG,CAAC;2BAAMC,IAAAA,wBAAa,EAACQ;;;QAClF;IACF;QAEK,mCAAA,4BAAA;;;YAAA,IAAMC,MAAN;gBAOHZ;YANA,IAAMa,SAASD,IAAIE,WAAW;gBACzB,kCAAA,2BAAA;;gBAAL,QAAK,YAAelE,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;oBAAvB,IAAMyD,QAAN;oBACHR,WAAWR,IAAI,CAAC,AAAC,GAAuEgB,OAArExD,YAAY,CAACwD,MAAM,EAAC,mDAAuD,OAANA,OAAM;oBAC9FN,YAAYV,IAAI,CAACc,IAAAA,wBAAa,EAACU;gBACjC;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILf,UAAUT,IAAI,CAAC,AAAC,IAA6F,OAA1FzC,OAAOsD,GAAG,CAAC,SAACG;uBAAU,AAAC,wCAA6C,OAANA,OAAM;eAASD,IAAI,CAAC,SAAQ;YAC7GJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGpD,OAAOsD,GAAG,CAAC;uBAAMC,IAAAA,wBAAa,EAACU;;QACpD;QARA,QAAK,aAAajC,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;QAAA;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEmC,UAAUlB,WAAWO,IAAI,CAAC;QAAQY,SAASlB,UAAUM,IAAI,CAAC,YAAY;QAAQa,QAAQ,AAAC,qBAAGlB,oBAAa,qBAAGC;IAAY;AACjI;AAIO,SAAerD,aAAaqB,IAAgB,EAAEX,KAAa,EAAE6D,IAAyB,EAAEjD,KAAoB;;YAuB7FkD,OAtBdC,aAIEC,WAAWC,WAAWC,WAAWC,OACmChD,aAApEC,OAAOC,WAAWC,aAAaC,YAAYC,SAASC,aAMtBc,oBAA9BmB,UAAUC,SAASC,QACrBQ,cACAC,UACAC,KAOAR,MACAS,MACAC;;;;oBAvBAT,cAAchE,oBAAoBC;oBACxC,IAAI+D,gBAAgB,MAAM;wBACxB,MAAM,IAAIU,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEV,OAAvBA,YAAYlE,KAAK,EAAC,OAAqDG,OAAhD+D,YAAY1D,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQ0D,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACmChD,cAAAA,WAAWnB,QAA/EoB,QAAoED,YAApEC,OAAOC,YAA6DF,YAA7DE,WAAWC,cAAkDH,YAAlDG,aAAaC,aAAqCJ,YAArCI,YAAYC,UAAyBL,YAAzBK,SAASC,cAAgBN,YAAhBM;oBAC5D,IAAIA,eAAgBL,MAAMW,MAAM,KAAK,KAAKR,WAAWQ,MAAM,KAAK,GAAI;;;;yBAEhEX,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMgB,eAAezB,MAAMC;;;oBAA3B;;;yBAClBQ,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMsD,IAAAA,gCAAiB,EAAC/D,MAAMC;;;oBAA9B;;;oBAEgB2B,qBAAAA,kBAAkBnB,OAAOC,WAAWC,aAAaC,aAA/EmC,WAA8BnB,mBAA9BmB,UAAUC,UAAoBpB,mBAApBoB,SAASC,SAAWrB,mBAAXqB;oBACrBQ,eAAe5C,QAAQO,MAAM,GAAG,IAAI,mDAAmD;oBACvFsC,WAAW7C,QAAQO,MAAM,GAAG,IAAI,KAAK;oBACrCuC,MAAM,AAAC,qBACuBZ,OADHlC,QAAQO,MAAM,GAAG,IAAI,2BAA2B,IAAG,iDAC3BqC,OAArBV,UAAS,cAEzCM,OAFqDI,cAAa,4BAG5DT,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KACoBI,OADjBH,WAAU,wCACgB,OAATG;oBACrB;;wBAAM1D,KAAKgE,OAAO,CAACL;;;oBAA1BR,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKc,GAAG,OAARd,OAAAA,AAAS,qBAAGF,eAAQ,qBAAIpC,QAAQO,MAAM,GAAG;4BAAUoC;;;;oBAAjEI,OAAQ;oBACRC,UAAUhD,QAAQO,MAAM,KAAK,IAAIwC,OAAOA,KAAKM,MAAM,CAAC,SAACC;+BAAQtD,QAAQuD,KAAK,CAAC,SAACnD;mCAAWrC,OAAOyF,IAAI,CAAC,SAAChC;oCAA8B8B;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAAC9B,MAAM,cAAV8B,wBAAAA,aAAc,IAAIlD;;;;oBAC1J;;wBAAO4C,QAAQU,KAAK,CAAC,GAAGf,OAAOtB,GAAG,CAAC;gCAAGsC,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAC3D;;AAIO,SAAS/F,mBAAmBuB,IAAgB;IACjD,OAAO;QACLyE,OAAO,SAAPA,MAAcpF,OAAO6D;;;;;;;gCAAsBlD;gCAAMX;gCAAO6D;;4BAAM;;gCAAMnD,SAASC;;;;;gCAA/CrB;oCAAgC;;;;;;;QAC9D+F,WAAW,SAAXA;mBAAiBhG,iBAAiBsB;;IACpC;AACF"}

package/dist/cjs/store/duckdb/native.d.cts

@@ -1,2 +1,4 @@
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
+export declare const DUCKDB_VERSION = "1.5.5-r.5";
+export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.5";
 export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/cjs/store/duckdb/native.d.ts

@@ -1,2 +1,4 @@
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
+export declare const DUCKDB_VERSION = "1.5.5-r.5";
+export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.5";
 export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/cjs/store/duckdb/native.js

@@ -9,19 +9,29 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get DUCKDB_INSTALL_SPEC () {
+        return DUCKDB_INSTALL_SPEC;
+    },
     get DUCKDB_PACKAGE () {
         return DUCKDB_PACKAGE;
     },
+    get DUCKDB_VERSION () {
+        return DUCKDB_VERSION;
+    },
     get duckdbApi () {
         return duckdbApi;
     }
 });
 var _nativets = require("../native.js");
 var DUCKDB_PACKAGE = '@duckdb/node-api';
+var DUCKDB_VERSION = '1.5.5-r.5';
+var DUCKDB_INSTALL_SPEC = "".concat(DUCKDB_PACKAGE, "@").concat(DUCKDB_VERSION);
 var DUCKDB = {
     store: 'duckdb',
     pkg: DUCKDB_PACKAGE,
-    sizeHint: '~110MB'
+    sizeHint: '~110MB',
+    version: DUCKDB_VERSION,
+    installSpec: DUCKDB_INSTALL_SPEC
 };
 var duckdbApiPromise;
 function duckdbApi() {

package/dist/cjs/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\nconst DUCKDB: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB' };\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["DUCKDB_PACKAGE","duckdbApi","DUCKDB","store","pkg","sizeHint","duckdbApiPromise","loadOrInstall","packageNodeModules"],"mappings":";;;;;;;;;;;QAEaA;eAAAA;;QAQGC;eAAAA;;;wBAVyD;AAElE,IAAMD,iBAAiB;AAE9B,IAAME,SAA2B;IAAEC,OAAO;IAAUC,KAAKJ;IAAgBK,UAAU;AAAS;AAE5F,IAAIC;AAIG,SAASL;IACd,IAAI,CAACK,kBAAkBA,mBAAmBC,IAAAA,uBAAa,EAAoCL,QAAQM,IAAAA,4BAAkB;IACrH,OAAOF;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_VERSION = '1.5.5-r.5';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  version: DUCKDB_VERSION,\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["DUCKDB_INSTALL_SPEC","DUCKDB_PACKAGE","DUCKDB_VERSION","duckdbApi","DUCKDB","store","pkg","sizeHint","version","installSpec","duckdbApiPromise","loadOrInstall","packageNodeModules"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QAFAC;eAAAA;;QACAC;eAAAA;;QAeGC;eAAAA;;;wBAlByD;AAElE,IAAMF,iBAAiB;AACvB,IAAMC,iBAAiB;AACvB,IAAMF,sBAAsB,AAAC,GAAoBE,OAAlBD,gBAAe,KAAkB,OAAfC;AAExD,IAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKL;IACLM,UAAU;IACVC,SAASN;IACTO,aAAaT;AACf;AAEA,IAAIU;AAIG,SAASP;IACd,IAAI,CAACO,kBAAkBA,mBAAmBC,IAAAA,uBAAa,EAAoCP,QAAQQ,IAAAA,4BAAkB;IACrH,OAAOF;AACT"}

package/dist/cjs/store/duckdb/ordered-bm25.d.cts

@@ -0,0 +1,17 @@
+import type { Connection } from '../types.js';
+export declare const ORDERED_BM25_MACRO = "sense_ordered_match_bm25";
+interface NativeMacroRow {
+    function_type: string;
+    parameters: unknown;
+    parameter_types: unknown;
+    macro_definition: string;
+}
+interface NativeMacroContract {
+    version: string;
+    macro: NativeMacroRow;
+}
+export declare function validateNativeMacroContract({ version, macro }: NativeMacroContract): string;
+export declare function ensureOrderedBm25(conn: Connection, state: {
+    orderedMacroReady: boolean;
+}): Promise<void>;
+export {};

package/dist/cjs/store/duckdb/ordered-bm25.d.ts

@@ -0,0 +1,17 @@
+import type { Connection } from '../types.js';
+export declare const ORDERED_BM25_MACRO = "sense_ordered_match_bm25";
+interface NativeMacroRow {
+    function_type: string;
+    parameters: unknown;
+    parameter_types: unknown;
+    macro_definition: string;
+}
+interface NativeMacroContract {
+    version: string;
+    macro: NativeMacroRow;
+}
+export declare function validateNativeMacroContract({ version, macro }: NativeMacroContract): string;
+export declare function ensureOrderedBm25(conn: Connection, state: {
+    orderedMacroReady: boolean;
+}): Promise<void>;
+export {};

package/dist/cjs/store/duckdb/ordered-bm25.js

@@ -0,0 +1,238 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get ORDERED_BM25_MACRO () {
+        return ORDERED_BM25_MACRO;
+    },
+    get ensureOrderedBm25 () {
+        return ensureOrderedBm25;
+    },
+    get validateNativeMacroContract () {
+        return validateNativeMacroContract;
+    }
+});
+var _nodecrypto = require("node:crypto");
+var _errorsts = require("../../errors.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+var ORDERED_BM25_MACRO = 'sense_ordered_match_bm25';
+var SUPPORTED_DUCKDB_VERSION = 'v1.5.5';
+var SUPPORTED_CONTRACT_SHA256 = '4476d543e43e7f8c89751cac0659b64e09d906cd3fefb1bfcaf4d15d3b1d9e00';
+var EXPECTED_PARAMETERS = [
+    'docname',
+    'query_string',
+    'fields',
+    'k',
+    'b',
+    'conjunctive'
+];
+var EXPECTED_DEFAULTS = {
+    fields: 'NULL',
+    k: '1.2',
+    b: '0.75',
+    conjunctive: 'false'
+};
+var EXPECTED_SIGNATURE = "docname, query_string, fields := ".concat(EXPECTED_DEFAULTS.fields, ", k := ").concat(EXPECTED_DEFAULTS.k, ", b := ").concat(EXPECTED_DEFAULTS.b, ", conjunctive := ").concat(EXPECTED_DEFAULTS.conjunctive);
+var UNORDERED_SUM = 'sum(subscore)';
+function unsupported(message) {
+    return new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "duckdb" cannot install its deterministic native BM25 adapter: '.concat(message, "; update the DuckDB adapter before using lexical search"));
+}
+function contractSha256(version, definition) {
+    return (0, _nodecrypto.createHash)('sha256').update("".concat(version, "\0").concat(EXPECTED_SIGNATURE, "\0").concat(definition)).digest('hex');
+}
+function validateNativeMacroContract(param) {
+    var version = param.version, macro = param.macro;
+    if (version !== SUPPORTED_DUCKDB_VERSION) throw unsupported("DuckDB ".concat(version, " is unsupported (validated version is ").concat(SUPPORTED_DUCKDB_VERSION, ")"));
+    if (macro.function_type !== 'macro') throw unsupported("match_bm25 is ".concat(macro.function_type, ", not a macro"));
+    if (JSON.stringify(macro.parameters) !== JSON.stringify(EXPECTED_PARAMETERS)) throw unsupported("match_bm25 parameters changed (".concat(JSON.stringify(macro.parameters), ")"));
+    if (!Array.isArray(macro.parameter_types) || macro.parameter_types.some(function(type) {
+        return type !== null;
+    })) throw unsupported('match_bm25 parameter types changed');
+    if (contractSha256(version, macro.macro_definition) !== SUPPORTED_CONTRACT_SHA256) throw unsupported('match_bm25 definition or reviewed parameter-default contract changed');
+    if (macro.macro_definition.split(UNORDERED_SUM).length !== 2) throw unsupported('match_bm25 does not contain exactly one validated unordered subscore sum');
+    return macro.macro_definition.replace(UNORDERED_SUM, 'sum(subscore ORDER BY subscore)');
+}
+function ensureOrderedBm25(conn, state) {
+    return _async_to_generator(function() {
+        var versionStatement, versionRow, macroStatement, macro, orderedDefinition;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (state.orderedMacroReady) return [
+                        2
+                    ];
+                    return [
+                        4,
+                        conn.prepare('SELECT version() AS version')
+                    ];
+                case 1:
+                    versionStatement = _state.sent();
+                    return [
+                        4,
+                        versionStatement.get()
+                    ];
+                case 2:
+                    versionRow = _state.sent();
+                    return [
+                        4,
+                        conn.prepare("\n    SELECT function_type, parameters, parameter_types, macro_definition\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content' AND function_name = 'match_bm25'\n  ")
+                    ];
+                case 3:
+                    macroStatement = _state.sent();
+                    return [
+                        4,
+                        macroStatement.get()
+                    ];
+                case 4:
+                    macro = _state.sent();
+                    if (!macro) throw unsupported('fts_main_content.match_bm25 is missing');
+                    orderedDefinition = validateNativeMacroContract({
+                        version: versionRow.version,
+                        macro: macro
+                    });
+                    return [
+                        4,
+                        conn.exec("CREATE OR REPLACE TEMP MACRO ".concat(ORDERED_BM25_MACRO, "(").concat(EXPECTED_SIGNATURE, ") AS ").concat(orderedDefinition))
+                    ];
+                case 5:
+                    _state.sent();
+                    state.orderedMacroReady = true;
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/duckdb/ordered-bm25.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { SenseError } from '../../errors.ts';\nimport type { Connection } from '../types.ts';\n\n// This is a connection-local TEMP macro. The contract hash covers the native fts definition,\n// DuckDB version, and the reviewed parameter defaults captured in the retained diagnostic.\nexport const ORDERED_BM25_MACRO = 'sense_ordered_match_bm25';\nconst SUPPORTED_DUCKDB_VERSION = 'v1.5.5';\nconst SUPPORTED_CONTRACT_SHA256 = '4476d543e43e7f8c89751cac0659b64e09d906cd3fefb1bfcaf4d15d3b1d9e00';\nconst EXPECTED_PARAMETERS = ['docname', 'query_string', 'fields', 'k', 'b', 'conjunctive'];\nconst EXPECTED_DEFAULTS = { fields: 'NULL', k: '1.2', b: '0.75', conjunctive: 'false' } as const;\nconst EXPECTED_SIGNATURE = `docname, query_string, fields := ${EXPECTED_DEFAULTS.fields}, k := ${EXPECTED_DEFAULTS.k}, b := ${EXPECTED_DEFAULTS.b}, conjunctive := ${EXPECTED_DEFAULTS.conjunctive}`;\nconst UNORDERED_SUM = 'sum(subscore)';\n\ninterface NativeMacroRow {\n  function_type: string;\n  parameters: unknown;\n  parameter_types: unknown;\n  macro_definition: string;\n}\n\ninterface NativeMacroContract {\n  version: string;\n  macro: NativeMacroRow;\n}\n\nfunction unsupported(message: string): SenseError {\n  return new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" cannot install its deterministic native BM25 adapter: ${message}; update the DuckDB adapter before using lexical search`);\n}\n\nfunction contractSha256(version: string, definition: string): string {\n  return createHash('sha256').update(`${version}\\0${EXPECTED_SIGNATURE}\\0${definition}`).digest('hex');\n}\n\nexport function validateNativeMacroContract({ version, macro }: NativeMacroContract): string {\n  if (version !== SUPPORTED_DUCKDB_VERSION) throw unsupported(`DuckDB ${version} is unsupported (validated version is ${SUPPORTED_DUCKDB_VERSION})`);\n  if (macro.function_type !== 'macro') throw unsupported(`match_bm25 is ${macro.function_type}, not a macro`);\n  if (JSON.stringify(macro.parameters) !== JSON.stringify(EXPECTED_PARAMETERS)) throw unsupported(`match_bm25 parameters changed (${JSON.stringify(macro.parameters)})`);\n  if (!Array.isArray(macro.parameter_types) || macro.parameter_types.some((type) => type !== null)) throw unsupported('match_bm25 parameter types changed');\n  if (contractSha256(version, macro.macro_definition) !== SUPPORTED_CONTRACT_SHA256) throw unsupported('match_bm25 definition or reviewed parameter-default contract changed');\n  if (macro.macro_definition.split(UNORDERED_SUM).length !== 2) throw unsupported('match_bm25 does not contain exactly one validated unordered subscore sum');\n  return macro.macro_definition.replace(UNORDERED_SUM, 'sum(subscore ORDER BY subscore)');\n}\n\nexport async function ensureOrderedBm25(conn: Connection, state: { orderedMacroReady: boolean }): Promise<void> {\n  if (state.orderedMacroReady) return;\n  const versionStatement = await conn.prepare('SELECT version() AS version');\n  const versionRow = (await versionStatement.get()) as { version: string };\n  const macroStatement = await conn.prepare(`\n    SELECT function_type, parameters, parameter_types, macro_definition\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content' AND function_name = 'match_bm25'\n  `);\n  const macro = (await macroStatement.get()) as NativeMacroRow | undefined;\n  if (!macro) throw unsupported('fts_main_content.match_bm25 is missing');\n  const orderedDefinition = validateNativeMacroContract({ version: versionRow.version, macro });\n  await conn.exec(`CREATE OR REPLACE TEMP MACRO ${ORDERED_BM25_MACRO}(${EXPECTED_SIGNATURE}) AS ${orderedDefinition}`);\n  state.orderedMacroReady = true;\n}\n"],"names":["ORDERED_BM25_MACRO","ensureOrderedBm25","validateNativeMacroContract","SUPPORTED_DUCKDB_VERSION","SUPPORTED_CONTRACT_SHA256","EXPECTED_PARAMETERS","EXPECTED_DEFAULTS","fields","k","b","conjunctive","EXPECTED_SIGNATURE","UNORDERED_SUM","unsupported","message","SenseError","contractSha256","version","definition","createHash","update","digest","macro","function_type","JSON","stringify","parameters","Array","isArray","parameter_types","some","type","macro_definition","split","length","replace","conn","state","versionStatement","versionRow","macroStatement","orderedDefinition","orderedMacroReady","prepare","get","exec"],"mappings":";;;;;;;;;;;QAMaA;eAAAA;;QAsCSC;eAAAA;;QAVNC;eAAAA;;;0BAlCW;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKpB,IAAMF,qBAAqB;AAClC,IAAMG,2BAA2B;AACjC,IAAMC,4BAA4B;AAClC,IAAMC,sBAAsB;IAAC;IAAW;IAAgB;IAAU;IAAK;IAAK;CAAc;AAC1F,IAAMC,oBAAoB;IAAEC,QAAQ;IAAQC,GAAG;IAAOC,GAAG;IAAQC,aAAa;AAAQ;AACtF,IAAMC,qBAAqB,AAAC,oCAAqEL,OAAlCA,kBAAkBC,MAAM,EAAC,WAAsCD,OAA7BA,kBAAkBE,CAAC,EAAC,WAAgDF,OAAvCA,kBAAkBG,CAAC,EAAC,qBAAiD,OAA9BH,kBAAkBI,WAAW;AAClM,IAAME,gBAAgB;AActB,SAASC,YAAYC,OAAe;IAClC,OAAO,IAAIC,oBAAU,CAAC,4BAA4B,AAAC,wEAA+E,OAARD,SAAQ;AACpI;AAEA,SAASE,eAAeC,OAAe,EAAEC,UAAkB;IACzD,OAAOC,IAAAA,sBAAU,EAAC,UAAUC,MAAM,CAAC,AAAC,GAAcT,OAAZM,SAAQ,MAA2BC,OAAvBP,oBAAmB,MAAe,OAAXO,aAAcG,MAAM,CAAC;AAChG;AAEO,SAASnB,4BAA4B,KAAuC;QAArCe,UAAF,MAAEA,SAASK,QAAX,MAAWA;IACrD,IAAIL,YAAYd,0BAA0B,MAAMU,YAAY,AAAC,UAAyDV,OAAhDc,SAAQ,0CAAiE,OAAzBd,0BAAyB;IAC/I,IAAImB,MAAMC,aAAa,KAAK,SAAS,MAAMV,YAAY,AAAC,iBAAoC,OAApBS,MAAMC,aAAa,EAAC;IAC5F,IAAIC,KAAKC,SAAS,CAACH,MAAMI,UAAU,MAAMF,KAAKC,SAAS,CAACpB,sBAAsB,MAAMQ,YAAY,AAAC,kCAAkE,OAAjCW,KAAKC,SAAS,CAACH,MAAMI,UAAU,GAAE;IACnK,IAAI,CAACC,MAAMC,OAAO,CAACN,MAAMO,eAAe,KAAKP,MAAMO,eAAe,CAACC,IAAI,CAAC,SAACC;eAASA,SAAS;QAAO,MAAMlB,YAAY;IACpH,IAAIG,eAAeC,SAASK,MAAMU,gBAAgB,MAAM5B,2BAA2B,MAAMS,YAAY;IACrG,IAAIS,MAAMU,gBAAgB,CAACC,KAAK,CAACrB,eAAesB,MAAM,KAAK,GAAG,MAAMrB,YAAY;IAChF,OAAOS,MAAMU,gBAAgB,CAACG,OAAO,CAACvB,eAAe;AACvD;AAEO,SAAeX,kBAAkBmC,IAAgB,EAAEC,KAAqC;;YAEvFC,kBACAC,YACAC,gBAKAlB,OAEAmB;;;;oBAVN,IAAIJ,MAAMK,iBAAiB,EAAE;;;oBACJ;;wBAAMN,KAAKO,OAAO,CAAC;;;oBAAtCL,mBAAmB;oBACL;;wBAAMA,iBAAiBM,GAAG;;;oBAAxCL,aAAc;oBACG;;wBAAMH,KAAKO,OAAO,CAAC;;;oBAApCH,iBAAiB;oBAKR;;wBAAMA,eAAeI,GAAG;;;oBAAjCtB,QAAS;oBACf,IAAI,CAACA,OAAO,MAAMT,YAAY;oBACxB4B,oBAAoBvC,4BAA4B;wBAAEe,SAASsB,WAAWtB,OAAO;wBAAEK,OAAAA;oBAAM;oBAC3F;;wBAAMc,KAAKS,IAAI,CAAC,AAAC,gCAAqDlC,OAAtBX,oBAAmB,KAA6ByC,OAA1B9B,oBAAmB,SAAyB,OAAlB8B;;;oBAAhG;oBACAJ,MAAMK,iBAAiB,GAAG;;;;;;IAC5B"}

package/dist/cjs/store/lock-wait.d.cts

@@ -1,2 +1,2 @@
-export declare function recordLockWaitMs(baseDir: string, ms: number): void;
-export declare function lockWaitBudgetMs(baseDir: string): number;
+export declare function recordLockWaitMs(configDir: string, ms: number): void;
+export declare function lockWaitBudgetMs(configDir: string): number;

package/dist/cjs/store/lock-wait.d.ts

@@ -1,2 +1,2 @@
-export declare function recordLockWaitMs(baseDir: string, ms: number): void;
-export declare function lockWaitBudgetMs(baseDir: string): number;
+export declare function recordLockWaitMs(configDir: string, ms: number): void;
+export declare function lockWaitBudgetMs(configDir: string): number;