sensemaking 0.18.3 → 0.18.4

package/dist/cjs/store/duckdb/lexical.js

@@ -356,11 +356,17 @@ function ensureFtsFresh(conn, state) {
 }
 // Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both
 // (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are
-// emitted -- positional `?` binding requires that order to match the assembled SQL text.
+// emitted -- positional `?` binding requires that order to match the assembled SQL text. Score
+// and gate params are collected apart and joined at the end because the assembled SQL puts
+// every score part ahead of every gate part: appending to one array term by term matches that
+// text only while the query has words or substrings, never both, and silently shifts each `?`
+// by one as soon as it has both (PRINCIPLES: no-silent-modes -- a shifted bind answers a
+// different question rather than failing).
 function buildScoreAndGate(words, substrings) {
     var scoreParts = [];
     var gateParts = [];
-    var params = [];
+    var scoreParams = [];
+    var gateParams = [];
     if (words.length > 0) {
         var wordQuery = words.join(' ');
         var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
@@ -368,7 +374,7 @@ function buildScoreAndGate(words, substrings) {
             for(var _iterator = FIELDS[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
                 var field = _step.value;
                 scoreParts.push("".concat(FIELD_WEIGHT[field], '.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := \'').concat(field, "', conjunctive := false), 0)"));
-                params.push(wordQuery);
+                scoreParams.push(wordQuery);
             }
         } catch (err) {
             _didIteratorError = true;
@@ -385,7 +391,7 @@ function buildScoreAndGate(words, substrings) {
             }
         }
         gateParts.push('fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL');
-        params.push(wordQuery);
+        gateParams.push(wordQuery);
     }
     var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
     try {
@@ -397,7 +403,7 @@ function buildScoreAndGate(words, substrings) {
                 for(var _iterator2 = FIELDS[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true){
                     var field1 = _step2.value;
                     scoreParts.push("".concat(FIELD_WEIGHT[field1], ".0 * CAST(contains(lower(content.").concat(field1, "), ?) AS INTEGER)"));
-                    params.push(needle);
+                    scoreParams.push(needle);
                 }
             } catch (err) {
                 _didIteratorError2 = true;
@@ -414,7 +420,7 @@ function buildScoreAndGate(words, substrings) {
                 }
             }
             gateParts.push("contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)");
-            params.push(needle);
+            gateParams.push(needle);
         }
     } catch (err) {
         _didIteratorError1 = true;
@@ -433,7 +439,7 @@ function buildScoreAndGate(words, substrings) {
     return {
         scoreSql: scoreParts.join(' + '),
         gateSql: gateParts.join(' AND '),
-        params: params
+        params: _to_consumable_array(scoreParams).concat(_to_consumable_array(gateParams))
     };
 }
 function queryLexical(conn, terms, opts, state) {

package/dist/cjs/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).\n// Checked against terms with quoted spans blanked -- those go through contains() and are\n// supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text.\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const params: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      params.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    params.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      params.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    params.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["createLexicalIndex","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","UNSPACED_SCRIPTS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","split","w","tok","ensureFtsFresh","conn","state","stale","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","params","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAyI9CA;eAAAA;;QA1BMC;eAAAA;;;wBA9GK;yBACM;6BACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,IAAMC,eAAe,IAAIC,OAAO,AAAC,IAAoB,OAAjBC,2BAAgB,EAAC,MAAI;AAEzD,uFAAuF;AACvF,+FAA+F;AAC/F,yFAAyF;AACzF,aAAa;AACb,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACQ,IAAIC;QACtD,IAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACnB,aAAasB,IAAI,CAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACpG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAahB,eAAee,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIpB,aAAasB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;iBACvCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,SAAeU,eAAeC,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;;;oBAClB;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACJ,MAAM;;;;;4CAC1B,uFAAuF;4CACvF,0FAA0F;4CAC1F,yFAAyF;4CACzF,oFAAoF;4CACpF;;gDAAMA,KAAKG,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBANA;oBAOAF,MAAMC,KAAK,GAAG;;;;;;IAChB;;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,yFAAyF;AACzF,SAASG,kBAAkBjB,KAAe,EAAEC,UAAoB;IAC9D,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,SAAoB,EAAE;IAE5B,IAAIpB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMgB,YAAYrB,MAAMsB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAe1C,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAM2C,QAAN;gBACHL,WAAWZ,IAAI,CAAC,AAAC,GAAgGiB,OAA9F1C,YAAY,CAAC0C,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHH,OAAOd,IAAI,CAACe;YACd;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILF,UAAUb,IAAI,CAAC;QACfc,OAAOd,IAAI,CAACe;IACd;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAapB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMuB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAe9C,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAM2C,SAAN;oBACHL,WAAWZ,IAAI,CAAC,AAAC,GAAyDiB,OAAvD1C,YAAY,CAAC0C,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFH,OAAOd,IAAI,CAACmB;gBACd;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILN,UAAUb,IAAI,CAAC;YACfc,OAAOd,IAAI,CAACmB;QACd;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUT,WAAWI,IAAI,CAAC;QAAQM,SAAST,UAAUG,IAAI,CAAC;QAAUF,QAAAA;IAAO;AACtF;AAKO,SAAezC,aAAaiC,IAAgB,EAAEpB,KAAa,EAAEqC,IAAyB,EAAEhB,KAAoB;;YAoBnGiB,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXpC,aAAtBC,OAAOC,YAKuBgB,oBAA9BU,UAAUC,SAASR,QACrBgB,KAOAN;;;;oBAlBAC,cAAcxC,oBAAoBC;oBACxC,IAAIuC,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAY1C,KAAK,EAAC,OAAqDG,OAAhDuC,YAAYlC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQkC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXpC,cAAAA,WAAWP,QAAjCQ,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAeC,MAAMC;;;oBAA3B;;;oBAEgBI,qBAAAA,kBAAkBjB,OAAOC,aAAvD0B,WAA8BV,mBAA9BU,UAAUC,UAAoBX,mBAApBW,SAASR,SAAWH,mBAAXG;oBACrBgB,MAAM,AAAC,wEAGTJ,OAFgCL,UAAS,sCAGnCC,OADNI,WAAU,gBAEVC,OADML,SAAQ,UACDM,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMtB,KAAK0B,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGV;4BAAQe;;;;oBAAlC;;wBAAQ;;;;IACV;;AAKO,SAASzD,mBAAmBkC,IAAgB;IACjD,IAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL0B,OAAO,SAAPA,MAAQhD,OAAOqC;mBAASlD,aAAaiC,MAAMpB,OAAOqC,MAAMhB;;QACxD4B,WAAW,SAAXA;YACE5B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).\n// Checked against terms with quoted spans blanked -- those go through contains() and are\n// supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text. Score\n// and gate params are collected apart and joined at the end because the assembled SQL puts\n// every score part ahead of every gate part: appending to one array term by term matches that\n// text only while the query has words or substrings, never both, and silently shifts each `?`\n// by one as soon as it has both (PRINCIPLES: no-silent-modes -- a shifted bind answers a\n// different question rather than failing).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["createLexicalIndex","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","UNSPACED_SCRIPTS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","split","w","tok","ensureFtsFresh","conn","state","stale","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QA+I9CA;eAAAA;;QA1BMC;eAAAA;;;wBApHK;yBACM;6BACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,IAAMC,eAAe,IAAIC,OAAO,AAAC,IAAoB,OAAjBC,2BAAgB,EAAC,MAAI;AAEzD,uFAAuF;AACvF,+FAA+F;AAC/F,yFAAyF;AACzF,aAAa;AACb,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACQ,IAAIC;QACtD,IAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACnB,aAAasB,IAAI,CAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACpG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAahB,eAAee,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIpB,aAAasB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;iBACvCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,SAAeU,eAAeC,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;;;oBAClB;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACJ,MAAM;;;;;4CAC1B,uFAAuF;4CACvF,0FAA0F;4CAC1F,yFAAyF;4CACzF,oFAAoF;4CACpF;;gDAAMA,KAAKG,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBANA;oBAOAF,MAAMC,KAAK,GAAG;;;;;;IAChB;;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,+FAA+F;AAC/F,2FAA2F;AAC3F,8FAA8F;AAC9F,8FAA8F;AAC9F,yFAAyF;AACzF,2CAA2C;AAC3C,SAASG,kBAAkBjB,KAAe,EAAEC,UAAoB;IAC9D,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMiB,YAAYtB,MAAMuB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAe3C,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAM4C,QAAN;gBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAgGkB,OAA9F3C,YAAY,CAAC2C,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYd,IAAI,CAACgB;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUb,IAAI,CAAC;QACfe,WAAWf,IAAI,CAACgB;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAarB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMwB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAe/C,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAM4C,SAAN;oBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAyDkB,OAAvD3C,YAAY,CAAC2C,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYd,IAAI,CAACoB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUb,IAAI,CAAC;YACfe,WAAWf,IAAI,CAACoB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAKO,SAAe1C,aAAaiC,IAAgB,EAAEpB,KAAa,EAAEuC,IAAyB,EAAElB,KAAoB;;YAoBnGmB,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXtC,aAAtBC,OAAOC,YAKuBgB,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc1C,oBAAoBC;oBACxC,IAAIyC,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAY5C,KAAK,EAAC,OAAqDG,OAAhDyC,YAAYpC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQoC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXtC,cAAAA,WAAWP,QAAjCQ,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAeC,MAAMC;;;oBAA3B;;;oBAEgBI,qBAAAA,kBAAkBjB,OAAOC,aAAvD2B,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMxB,KAAK4B,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAKO,SAAS3D,mBAAmBkC,IAAgB;IACjD,IAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,SAAPA,MAAQlD,OAAOuC;mBAASpD,aAAaiC,MAAMpB,OAAOuC,MAAMlB;;QACxD8B,WAAW,SAAXA;YACE9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}

package/dist/esm/store/duckdb/lexical.js

@@ -96,33 +96,42 @@ async function ensureFtsFresh(conn, state) {
 }
 // Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both
 // (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are
-// emitted -- positional `?` binding requires that order to match the assembled SQL text.
+// emitted -- positional `?` binding requires that order to match the assembled SQL text. Score
+// and gate params are collected apart and joined at the end because the assembled SQL puts
+// every score part ahead of every gate part: appending to one array term by term matches that
+// text only while the query has words or substrings, never both, and silently shifts each `?`
+// by one as soon as it has both (PRINCIPLES: no-silent-modes -- a shifted bind answers a
+// different question rather than failing).
 function buildScoreAndGate(words, substrings) {
     const scoreParts = [];
     const gateParts = [];
-    const params = [];
+    const scoreParams = [];
+    const gateParams = [];
     if (words.length > 0) {
         const wordQuery = words.join(' ');
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
-            params.push(wordQuery);
+            scoreParams.push(wordQuery);
         }
         gateParts.push(`fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`);
-        params.push(wordQuery);
+        gateParams.push(wordQuery);
     }
     for (const raw of substrings){
         const needle = raw.toLowerCase();
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);
-            params.push(needle);
+            scoreParams.push(needle);
         }
         gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);
-        params.push(needle);
+        gateParams.push(needle);
     }
     return {
         scoreSql: scoreParts.join(' + '),
         gateSql: gateParts.join(' AND '),
-        params
+        params: [
+            ...scoreParams,
+            ...gateParams
+        ]
     };
 }
 // Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).\n// Checked against terms with quoted spans blanked -- those go through contains() and are\n// supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text.\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const params: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      params.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    params.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      params.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    params.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","UNSPACED_SCRIPTS","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","params","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAER,iBAAiB,CAAC,CAAC,EAAE;AAEzD,uFAAuF;AACvF,+FAA+F;AAC/F,yFAAyF;AACzF,aAAa;AACb,MAAMS,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAClB,aAAaqB,IAAI,CAACH,SAAS;YAAA,KAAK,MAAMI,KAAKJ,OAAOK,KAAK,CAAC,OAAQ,IAAID,EAAEH,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACE;QAAE;QACpG,OAAO;IACT;IACA,KAAK,MAAME,OAAOjB,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIL,MAAM,KAAK,GAAG;QACtB,IAAInB,aAAaqB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;aACvCV,MAAMM,IAAI,CAACI;IAClB;IACA,OAAO;QAAEV;QAAOC;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,eAAeU,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMnC,gBAAgBgC,MAAM;QAC1B,uFAAuF;QACvF,0FAA0F;QAC1F,yFAAyF;QACzF,oFAAoF;QACpF,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,yFAAyF;AACzF,SAASE,kBAAkBhB,KAAe,EAAEC,UAAoB;IAC9D,MAAMgB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,SAAoB,EAAE;IAE5B,IAAInB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMe,YAAYpB,MAAMqB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAASzC,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACwC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJH,OAAOb,IAAI,CAACc;QACd;QACAF,UAAUZ,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGa,OAAOb,IAAI,CAACc;IACd;IAEA,KAAK,MAAMG,OAAOtB,WAAY;QAC5B,MAAMuB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAASzC,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACwC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGH,OAAOb,IAAI,CAACkB;QACd;QACAN,UAAUZ,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHa,OAAOb,IAAI,CAACkB;IACd;IAEA,OAAO;QAAEE,UAAUT,WAAWI,IAAI,CAAC;QAAQM,SAAST,UAAUG,IAAI,CAAC;QAAUF;IAAO;AACtF;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,mGAAmG;AACnG,OAAO,eAAeS,aAAahB,IAAgB,EAAEpB,KAAa,EAAEqC,IAAyB,EAAEhB,KAAoB;IACjH,MAAMiB,cAAcvC,oBAAoBC;IACxC,IAAIsC,gBAAgB,MAAM;QACxB,MAAM,IAAIpD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEoD,YAAYzC,KAAK,CAAC,GAAG,EAAEyC,YAAYjC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEiC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE7B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMM,eAAeC,MAAMC;IAEjD,MAAM,EAAEa,QAAQ,EAAEC,OAAO,EAAER,MAAM,EAAE,GAAGH,kBAAkBhB,OAAOC;IAC/D,MAAMkC,MAAM,CAAC;oCACqB,EAAET,SAAS;;IAE3C,EAAEK,UAAU;UACN,EAAEJ,QAAQ;IAChB,EAAEK,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAMxB,KAAKyB,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAInB,QAAQe;AACpC;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,0FAA0F;AAC1F,OAAO,SAASK,mBAAmB3B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL0B,OAAO,CAAChD,OAAOqC,OAASD,aAAahB,MAAMpB,OAAOqC,MAAMhB;QACxD4B,WAAW;YACT5B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).\n// Checked against terms with quoted spans blanked -- those go through contains() and are\n// supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text. Score\n// and gate params are collected apart and joined at the end because the assembled SQL puts\n// every score part ahead of every gate part: appending to one array term by term matches that\n// text only while the query has words or substrings, never both, and silently shifts each `?`\n// by one as soon as it has both (PRINCIPLES: no-silent-modes -- a shifted bind answers a\n// different question rather than failing).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","UNSPACED_SCRIPTS","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAER,iBAAiB,CAAC,CAAC,EAAE;AAEzD,uFAAuF;AACvF,+FAA+F;AAC/F,yFAAyF;AACzF,aAAa;AACb,MAAMS,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAClB,aAAaqB,IAAI,CAACH,SAAS;YAAA,KAAK,MAAMI,KAAKJ,OAAOK,KAAK,CAAC,OAAQ,IAAID,EAAEH,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACE;QAAE;QACpG,OAAO;IACT;IACA,KAAK,MAAME,OAAOjB,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIL,MAAM,KAAK,GAAG;QACtB,IAAInB,aAAaqB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;aACvCV,MAAMM,IAAI,CAACI;IAClB;IACA,OAAO;QAAEV;QAAOC;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,eAAeU,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMnC,gBAAgBgC,MAAM;QAC1B,uFAAuF;QACvF,0FAA0F;QAC1F,yFAAyF;QACzF,oFAAoF;QACpF,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,+FAA+F;AAC/F,2FAA2F;AAC3F,8FAA8F;AAC9F,8FAA8F;AAC9F,yFAAyF;AACzF,2CAA2C;AAC3C,SAASE,kBAAkBhB,KAAe,EAAEC,UAAoB;IAC9D,MAAMgB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIpB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMgB,YAAYrB,MAAMsB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAAS1C,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACyC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYb,IAAI,CAACe;QACnB;QACAH,UAAUZ,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGc,WAAWd,IAAI,CAACe;IAClB;IAEA,KAAK,MAAMG,OAAOvB,WAAY;QAC5B,MAAMwB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAAS1C,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACyC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYb,IAAI,CAACmB;QACnB;QACAP,UAAUZ,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHc,WAAWd,IAAI,CAACmB;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,mGAAmG;AACnG,OAAO,eAAeU,aAAalB,IAAgB,EAAEpB,KAAa,EAAEuC,IAAyB,EAAElB,KAAoB;IACjH,MAAMmB,cAAczC,oBAAoBC;IACxC,IAAIwC,gBAAgB,MAAM;QACxB,MAAM,IAAItD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEsD,YAAY3C,KAAK,CAAC,GAAG,EAAE2C,YAAYnC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEmC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE/B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMM,eAAeC,MAAMC;IAEjD,MAAM,EAAEc,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBhB,OAAOC;IAC/D,MAAMoC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAM1B,KAAK2B,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,0FAA0F;AAC1F,OAAO,SAASK,mBAAmB7B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,CAAClD,OAAOuC,OAASD,aAAalB,MAAMpB,OAAOuC,MAAMlB;QACxD8B,WAAW;YACT9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.18.3",
+  "version": "0.18.4",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",