sensemaking 0.23.1 → 0.24.0

package/dist/esm/store/duckdb/lexical.js

@@ -1,7 +1,8 @@
 // D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /
-// unspaced scripts, JS excerpts for every doc (no snippet()).
+// unspaced scripts, JS snippets for every doc (no snippet()).
 import { SenseError } from '../../errors.js';
-import { hasUnspacedRun } from '../../text/segment.js';
+import { foldForSearch, matchesSearchPhrase, searchTokens } from '../../text/segment.js';
+import { getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 const FIELDS = [
     'title',
@@ -50,50 +51,79 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
+// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of
+// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.
+const FTS_STALE_META_KEY = 'fts_stale';
 // One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
 // stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
 const ftsState = new WeakMap();
-function stateFor(conn) {
+async function stateFor(conn) {
     let state = ftsState.get(conn);
     if (!state) {
+        // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that
+        // never built an index, or one a prior process marked stale and never got to clear.
         state = {
-            stale: true
+            stale: await getMeta(conn, FTS_STALE_META_KEY) !== '0'
         };
         ftsState.set(conn, state);
     }
     return state;
 }
-// Called whenever this store's reconcileContent changes `content`, so the next lexical query
-// rebuilds instead of searching stale content.
-export function markContentStale(conn) {
-    stateFor(conn).stale = true;
+// Called whenever this store's reconcileContent changes `content`; must run inside that same
+// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.
+export async function markContentStale(conn) {
+    ftsState.set(conn, {
+        stale: true
+    });
+    await setMeta(conn, FTS_STALE_META_KEY, '1');
 }
-// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
-// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
+// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,
+// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.
 function splitTerms(terms) {
     const words = [];
+    const bareWords = [];
+    const phraseWords = [];
     const substrings = [];
+    const phrases = [];
+    let emptyPhrase = false;
     const withoutPhrases = terms.replace(/"([^"]*)"/g, (_m, inner)=>{
         const phrase = inner.trim();
-        if (phrase.length === 0) return ' ';
-        substrings.push(phrase);
-        if (!hasUnspacedRun(phrase)) {
-            for (const w of phrase.split(/\s+/))if (w.length > 0) words.push(w);
+        const phraseTokens = searchTokens(phrase);
+        if (phraseTokens.length === 0) {
+            emptyPhrase = true;
+            return ' ';
+        }
+        phrases.push(phraseTokens);
+        for (const token of phraseTokens){
+            if (token.unspaced) substrings.push(token.text);
+            else {
+                words.push(token.text);
+                phraseWords.push(token.text);
+            }
         }
         return ' ';
     });
     for (const tok of withoutPhrases.split(/\s+/)){
         if (tok.length === 0) continue;
-        if (hasUnspacedRun(tok)) substrings.push(tok);
-        else words.push(tok);
+        for (const token of searchTokens(tok)){
+            if (token.unspaced) substrings.push(token.text);
+            else {
+                words.push(token.text);
+                bareWords.push(token.text);
+            }
+        }
     }
     return {
         words,
-        substrings
+        bareWords,
+        phraseWords,
+        substrings,
+        phrases,
+        emptyPhrase
     };
 }
-// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only
-// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).
+// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays
+// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.
 async function ensureFtsFresh(conn, state) {
     if (!state.stale) return;
     await conn.exec('INSTALL fts; LOAD fts;');
@@ -101,70 +131,94 @@ async function ensureFtsFresh(conn, state) {
         // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's
         // default 571-word list would otherwise drop common words (e.g. "and") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.
         await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);
+        // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant
+        // rebuild next time), never a cleared mark serving a half-built index as fresh.
+        await setMeta(conn, FTS_STALE_META_KEY, '0');
     });
     state.stale = false;
 }
 // Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
 // score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
-function buildScoreAndGate(words, substrings) {
+function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
     const scoreParts = [];
     const gateParts = [];
     const scoreParams = [];
     const gateParams = [];
     if (words.length > 0) {
-        const wordQuery = words.join(' ');
+        const wordQuery = words.map(foldForSearch).join(' ');
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
             scoreParams.push(wordQuery);
         }
-        gateParts.push(`fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`);
-        gateParams.push(wordQuery);
+        if (bareWords.length > 0) {
+            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            gateParts.push(nativeGate);
+            gateParams.push(bareWords.map(foldForSearch).join(' '));
+        }
+        if (phraseWords.length > 0) {
+            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const fallback = phraseWords.map((_word)=>`(${FIELDS.map((field)=>`contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');
+            gateParts.push(`(${nativeGate} OR (${fallback}))`);
+            gateParams.push(phraseWords.map(foldForSearch).join(' '));
+            gateParams.push(...phraseWords.flatMap((word)=>FIELDS.map(()=>foldForSearch(word))));
+        }
     }
     for (const raw of substrings){
         const needle = raw.toLowerCase();
         for (const field of FIELDS){
-            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);
-            scoreParams.push(needle);
+            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);
+            scoreParams.push(foldForSearch(needle));
         }
-        gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);
-        gateParams.push(needle);
+        gateParts.push(`(${FIELDS.map((field)=>`contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);
+        gateParams.push(...FIELDS.map(()=>foldForSearch(needle)));
     }
     return {
         scoreSql: scoreParts.join(' + '),
-        gateSql: gateParts.join(' AND '),
+        gateSql: gateParts.join(' AND ') || 'TRUE',
         params: [
             ...scoreParams,
             ...gateParams
         ]
     };
 }
-// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).
-// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).
+// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).
+// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.
 export async function queryLexical(conn, terms, opts, state) {
     const unsupported = unsupportedOperator(terms);
     if (unsupported !== null) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "duckdb" does not implement FTS5's ${unsupported.label} ("${unsupported.token}") in this build; rephrase "${terms.trim()}" without it, or set "store" to "sqlite" in this tree's config to search it as written`);
     }
     const { whereJoin, whereCond, scopeCond, limit } = opts;
-    const { words, substrings } = splitTerms(terms);
-    if (words.length === 0 && substrings.length === 0) return [];
+    const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);
+    if (emptyPhrase || words.length === 0 && substrings.length === 0) return [];
     if (words.length > 0) await ensureFtsFresh(conn, state);
-    const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);
-    const sql = `SELECT path, NULL AS hit FROM (
-    SELECT content."path" AS path, (${scoreSql}) AS score
+    const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);
+    const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
+    const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';
+    const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (
+    SELECT content."path" AS path, (${scoreSql}) AS score${selectFields}
     FROM content
     ${whereJoin}
     WHERE ${gateSql}
     ${whereCond} ${scopeCond}
-  ) sq ORDER BY score DESC, path LIMIT ?`;
+    ) sq ORDER BY score DESC, path${sqlLimit}`;
     const stmt = await conn.prepare(sql);
-    return await stmt.all(...params, limit);
+    const rows = await stmt.all(...params, ...phrases.length > 0 ? [] : [
+        limit
+    ]);
+    const matches = phrases.length === 0 ? rows : rows.filter((row)=>phrases.every((phrase)=>FIELDS.some((field)=>{
+                var _row_field;
+                return matchesSearchPhrase((_row_field = row[field]) !== null && _row_field !== void 0 ? _row_field : '', phrase);
+            })));
+    return matches.slice(0, limit).map(({ path })=>({
+            path
+        }));
 }
-// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches
-// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.
+// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know
+// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.
 export function createLexicalIndex(conn) {
     return {
-        query: (terms, opts)=>queryLexical(conn, terms, opts, stateFor(conn)),
+        query: async (terms, opts)=>queryLexical(conn, terms, opts, await stateFor(conn)),
         markStale: ()=>markContentStale(conn)
     };
 }

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nfunction stateFor(conn: Connection): FtsIndexState {\n  let state = ftsState.get(conn);\n  if (!state) {\n    state = { stale: true };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`, so the next lexical query\n// rebuilds instead of searching stale content.\nexport function markContentStale(conn: Connection): void {\n  stateFor(conn).stale = true;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches\n// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","hasUnspacedRun","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","ftsState","WeakMap","stateFor","conn","state","get","stale","set","markContentStale","splitTerms","words","substrings","_m","inner","phrase","length","push","w","split","tok","ensureFtsFresh","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,SAASC,SAASC,IAAgB;IAChC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;QAAK;QACtBN,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,6FAA6F;AAC7F,+CAA+C;AAC/C,OAAO,SAASI,iBAAiBL,IAAgB;IAC/CD,SAASC,MAAMG,KAAK,GAAG;AACzB;AAEA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASG,WAAWhB,KAAa;IAC/B,MAAMiB,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMjB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACiB,IAAIC;QACtD,MAAMC,SAASD,MAAMd,IAAI;QACzB,IAAIe,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAChC,eAAegC,SAAS;YAAA,KAAK,MAAMG,KAAKH,OAAOI,KAAK,CAAC,OAAQ,IAAID,EAAEF,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACC;QAAE;QACjG,OAAO;IACT;IACA,KAAK,MAAME,OAAOzB,eAAewB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACtB,IAAIjC,eAAeqC,MAAMR,WAAWK,IAAI,CAACG;aACpCT,MAAMM,IAAI,CAACG;IAClB;IACA,OAAO;QAAET;QAAOC;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,eAAeS,eAAejB,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAME,KAAK,EAAE;IAClB,MAAMH,KAAKkB,IAAI,CAAC;IAChB,MAAMtC,gBAAgBoB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKkB,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAjB,MAAME,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASgB,kBAAkBZ,KAAe,EAAEC,UAAoB;IAC9D,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIhB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMY,YAAYjB,MAAMkB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAAS7C,OAAQ;YAC1BuC,WAAWP,IAAI,CAAC,GAAG/B,YAAY,CAAC4C,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYT,IAAI,CAACW;QACnB;QACAH,UAAUR,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGU,WAAWV,IAAI,CAACW;IAClB;IAEA,KAAK,MAAMG,OAAOnB,WAAY;QAC5B,MAAMoB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAAS7C,OAAQ;YAC1BuC,WAAWP,IAAI,CAAC,GAAG/B,YAAY,CAAC4C,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYT,IAAI,CAACe;QACnB;QACAP,UAAUR,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHU,WAAWV,IAAI,CAACe;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,kHAAkH;AAClH,yIAAyI;AACzI,OAAO,eAAeU,aAAajC,IAAgB,EAAEV,KAAa,EAAE4C,IAAyB,EAAEjC,KAAoB;IACjH,MAAMkC,cAAc9C,oBAAoBC;IACxC,IAAI6C,gBAAgB,MAAM;QACxB,MAAM,IAAIzD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEyD,YAAYhD,KAAK,CAAC,GAAG,EAAEgD,YAAYxC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEwC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE3B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWhB;IACzC,IAAIiB,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMK,eAAejB,MAAMC;IAEjD,MAAM,EAAE6B,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBZ,OAAOC;IAC/D,MAAMgC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAMzC,KAAK0C,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,2FAA2F;AAC3F,gGAAgG;AAChG,OAAO,SAASK,mBAAmB5C,IAAgB;IACjD,OAAO;QACL6C,OAAO,CAACvD,OAAO4C,OAASD,aAAajC,MAAMV,OAAO4C,MAAMnC,SAASC;QACjE8C,WAAW,IAAMzC,iBAAiBL;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0' };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","set","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","ensureFtsFresh","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","stmt","prepare","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM1B,QAAQuB,MAAMJ,wBAAyB;QAAI;QACnEC,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeI,iBAAiBL,IAAgB;IACrDH,SAASO,GAAG,CAACJ,MAAM;QAAEG,OAAO;IAAK;IACjC,MAAMzB,QAAQsB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASU,WAAWjB,KAAa;IAC/B,MAAMkB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAMtB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACsB,IAAIC;QACtD,MAAMC,SAASD,MAAMnB,IAAI;QACzB,MAAMqB,eAAexC,aAAauC;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAMtB,SAASsB,aAAc;YAChC,IAAItB,MAAMyB,QAAQ,EAAET,WAAWQ,IAAI,CAACxB,MAAMV,IAAI;iBACzC;gBACHuB,MAAMW,IAAI,CAACxB,MAAMV,IAAI;gBACrByB,YAAYS,IAAI,CAACxB,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMoC,OAAO9B,eAAe+B,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMvB,SAASlB,aAAa4C,KAAM;YACrC,IAAI1B,MAAMyB,QAAQ,EAAET,WAAWQ,IAAI,CAACxB,MAAMV,IAAI;iBACzC;gBACHuB,MAAMW,IAAI,CAACxB,MAAMV,IAAI;gBACrBwB,UAAUU,IAAI,CAACxB,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAEuB;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,eAAetB,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAME,KAAK,EAAE;IAClB,MAAMH,KAAKuB,IAAI,CAAC;IAChB,MAAM5C,gBAAgBqB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKuB,IAAI,CAAC,CAAC,qGAAqG,CAAC;QACvH,0FAA0F;QAC1F,gFAAgF;QAChF,MAAM7C,QAAQsB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASqB,kBAAkBjB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,MAAMe,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMU,MAAM,GAAG,GAAG;QACpB,MAAMY,YAAYtB,MAAMuB,GAAG,CAACxD,eAAeyD,IAAI,CAAC;QAChD,KAAK,MAAMC,SAASpD,OAAQ;YAC1B6C,WAAWP,IAAI,CAAC,GAAGrC,YAAY,CAACmD,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJL,YAAYT,IAAI,CAACW;QACnB;QACA,IAAIrB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMgB,aAAa,CAAC,+EAA+E,CAAC;YACpGP,UAAUR,IAAI,CAACe;YACfL,WAAWV,IAAI,CAACV,UAAUsB,GAAG,CAACxD,eAAeyD,IAAI,CAAC;QACpD;QACA,IAAItB,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMgB,aAAa,CAAC,+EAA+E,CAAC;YACpG,MAAMC,WAAWzB,YAAYqB,GAAG,CAAC,CAACK,QAAU,CAAC,CAAC,EAAEvD,OAAOkD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJL,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEe,WAAW,KAAK,EAAEC,SAAS,EAAE,CAAC;YACjDN,WAAWV,IAAI,CAACT,YAAYqB,GAAG,CAACxD,eAAeyD,IAAI,CAAC;YACpDH,WAAWV,IAAI,IAAIT,YAAY2B,OAAO,CAAC,CAACC,OAASzD,OAAOkD,GAAG,CAAC,IAAMxD,cAAc+D;QAClF;IACF;IAEA,KAAK,MAAMC,OAAO5B,WAAY;QAC5B,MAAM6B,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMR,SAASpD,OAAQ;YAC1B6C,WAAWP,IAAI,CAAC,GAAGrC,YAAY,CAACmD,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHL,YAAYT,IAAI,CAAC5C,cAAciE;QACjC;QACAb,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEtC,OAAOkD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GH,WAAWV,IAAI,IAAItC,OAAOkD,GAAG,CAAC,IAAMxD,cAAciE;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWM,IAAI,CAAC;QAAQW,SAAShB,UAAUK,IAAI,CAAC,YAAY;QAAQY,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAa5C,IAAgB,EAAEX,KAAa,EAAEwD,IAAyB,EAAE5C,KAAoB;IACjH,MAAM6C,cAAc1D,oBAAoBC;IACxC,IAAIyD,gBAAgB,MAAM;QACxB,MAAM,IAAIzE,WAAW,4BAA4B,CAAC,yCAAyC,EAAEyE,YAAY5D,KAAK,CAAC,GAAG,EAAE4D,YAAYpD,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEoD,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAEtC,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWjB;IACvF,IAAIuB,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMK,eAAetB,MAAMC;IAEjD,MAAM,EAAEwC,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGnB,kBAAkBjB,OAAOC,WAAWC,aAAaC;IACvF,MAAMyC,eAAexC,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,WAAWzC,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAMoC,MAAM,CAAC,kBAAkB,EAAE1C,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAEwB,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAME,OAAO,MAAMtD,KAAKuD,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAId,WAAYhC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACiC;KAAM;IAC7E,MAAMQ,UAAU/C,QAAQM,MAAM,KAAK,IAAIuC,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQjD,QAAQkD,KAAK,CAAC,CAAC9C,SAAWnC,OAAOkF,IAAI,CAAC,CAAC9B;oBAA8B4B;uBAApBrF,qBAAoBqF,aAAAA,GAAG,CAAC5B,MAAM,cAAV4B,wBAAAA,aAAc,IAAI7C;;IAC1J,OAAO2C,QAAQK,KAAK,CAAC,GAAGb,OAAOpB,GAAG,CAAC,CAAC,EAAEkC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmBjE,IAAgB;IACjD,OAAO;QACLkE,OAAO,OAAO7E,OAAOwD,OAASD,aAAa5C,MAAMX,OAAOwD,MAAM,MAAM9C,SAASC;QAC7EmE,WAAW,IAAM9D,iBAAiBL;IACpC;AACF"}

package/dist/esm/store/duckdb/reconcile.js

@@ -33,8 +33,8 @@ async function reconcileContent(conn, touched, docs, _delta) {
     // connection, so no second writer can have landed one of these rows: nothing can conflict.
     await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));
     // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
-    // not here (lexical.ts's FtsIndexState).
-    markContentStale(conn);
+    // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.
+    await markContentStale(conn);
 }
 // DuckDB rejects more than one ALTER command per statement ("Parser Error: Only one ALTER
 // command per statement is supported", measured), so every name's clause joins into one string

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","appendRows","quoteIdent","markContentStale","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","addColumns","names","exec","name","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AAEtD,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,MAAMC,qBAAqB,CAAC,qBAAqB,EAAED,gBAAgBE,GAAG,CAACJ,YAAYK,IAAI,CAAC,MAAM,qBAAqB,CAAC;AAEpH,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQZ,GAAG,CAAC,CAACiB,IAAM;YAACA;SAAE;IAE1B,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAMtB,WAAWgB,MAAM,WAAWb,iBAAiBC,oBAAoBc,KAAKb,GAAG,CAACE;IAChF,6FAA6F;IAC7F,yCAAyC;IACzCL,iBAAiBc;AACnB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeO,WAAWP,IAAgB,EAAEQ,KAAe;IACzD,IAAIA,MAAMJ,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKS,IAAI,CAACD,MAAMnB,GAAG,CAAC,CAACqB,OAAS,CAAC,mCAAmC,EAAEzB,WAAWyB,MAAM,QAAQ,CAAC,EAAEpB,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMqB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhB,yBAAyB;YACnC,MAAM,IAAIf,WAAW,gBAAgB,CAAC,uBAAuB,EAAE+B,MAAM,8CAA8C,EAAEhB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAS;IACAR;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","appendRows","quoteIdent","markContentStale","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","addColumns","names","exec","name","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AAEtD,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,MAAMC,qBAAqB,CAAC,qBAAqB,EAAED,gBAAgBE,GAAG,CAACJ,YAAYK,IAAI,CAAC,MAAM,qBAAqB,CAAC;AAEpH,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQZ,GAAG,CAAC,CAACiB,IAAM;YAACA;SAAE;IAE1B,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAMtB,WAAWgB,MAAM,WAAWb,iBAAiBC,oBAAoBc,KAAKb,GAAG,CAACE;IAChF,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAML,iBAAiBc;AACzB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeO,WAAWP,IAAgB,EAAEQ,KAAe;IACzD,IAAIA,MAAMJ,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKS,IAAI,CAACD,MAAMnB,GAAG,CAAC,CAACqB,OAAS,CAAC,mCAAmC,EAAEzB,WAAWyB,MAAM,QAAQ,CAAC,EAAEpB,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMqB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhB,yBAAyB;YACnC,MAAM,IAAIf,WAAW,gBAAgB,CAAC,uBAAuB,EAAE+B,MAAM,8CAA8C,EAAEhB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAS;IACAR;AACF,EAAE"}

package/dist/esm/store/duckdb/store.js

@@ -7,12 +7,10 @@ import { fieldStats } from './fieldStats.js';
 import { createLexicalIndex } from './lexical.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 // Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native
-// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.
+// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. segment() registers as a UDF (sql-functions.ts).
 export const CAPABILITIES = new Set([
-    'lexical',
-    'phrases',
     'vectors',
-    'sql-functions'
+    'segment'
 ]);
 // Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.
 // The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.

package/dist/esm/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB,EAAE;AAEjH,mKAAmK;AACnK,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,MAAMC,MAAMV,mBAAmBS;IAC/B,OAAO;QACLE,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBc,MAAMU;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWe;iBAAO;YACtC;YACAV,YAAY,CAACsB,SAASC,aAAevB,WAAWU,MAAMY,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOd,IAAIc,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAM7B,YAAYY;YAC3BkB,cAAc,CAACC,OAASzB,iBAAiBK,QAAQC,MAAMhB,YAAYmC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,QAAQsB,IAAIrC,YAAYuC,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAASlC,YAAYM,QAAQC,MAAMhB,YAAY0C,MAAMC;YACrEC,WAAW,CAACF,OAASvC,aAAaa,MAAM0B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAMxB,SAAQD,GAAW;gBACvB,MAAM0B,WAAW,MAAMhC,OAAOO,OAAO,CAACD;gBACtC,OAAO;oBACLO,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEjC,MAAM6B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAMtD,aAAa8B,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFjD,OAAOkD,cAAc;YACrBnD,SAASoD,SAAS;QACpB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. segment() registers as a UDF (sql-functions.ts).\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors', 'segment']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,oHAAoH;AACpH,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;CAAU,EAAE;AAErF,mKAAmK;AACnK,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,MAAMC,MAAMV,mBAAmBS;IAC/B,OAAO;QACLE,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBc,MAAMU;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWe;iBAAO;YACtC;YACAV,YAAY,CAACsB,SAASC,aAAevB,WAAWU,MAAMY,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOd,IAAIc,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAM7B,YAAYY;YAC3BkB,cAAc,CAACC,OAASzB,iBAAiBK,QAAQC,MAAMhB,YAAYmC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,QAAQsB,IAAIrC,YAAYuC,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAASlC,YAAYM,QAAQC,MAAMhB,YAAY0C,MAAMC;YACrEC,WAAW,CAACF,OAASvC,aAAaa,MAAM0B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAMxB,SAAQD,GAAW;gBACvB,MAAM0B,WAAW,MAAMhC,OAAOO,OAAO,CAACD;gBACtC,OAAO;oBACLO,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEjC,MAAM6B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAMtD,aAAa8B,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFjD,OAAOkD,cAAc;YACrBnD,SAASoD,SAAS;QACpB;IACF;AACF"}

package/dist/esm/store/duckdb/vectors.js

@@ -50,7 +50,7 @@ export async function writeVectorBatch(duckdb, conn, dims, rows) {
         }
     });
 }
-// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks the winning chunk's line range from the same row max(score) came from, per path.
+// Best chunk per file by cosine, with the earliest chunk winning exact score ties.
 // An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.
 export async function scanCandidates(duckdb, qv, dims, fetch, allowed) {
     if (allowed && allowed.size === 0) return [];
@@ -58,24 +58,35 @@ export async function scanCandidates(duckdb, qv, dims, fetch, allowed) {
     const allowedList = allowed ? [
         ...allowed
     ] : [];
-    const sql = `SELECT "path", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score
+    const query = padded(qv, dims);
+    const queryNorm = Math.sqrt(query.reduce((sum, value)=>sum + value * value, 0));
+    const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';
+    const sql = `SELECT "path", start_line, end_line, score
     FROM (
-      SELECT "path", start_line, end_line, array_cosine_similarity(vector, ?) AS score
-      FROM embeddings
-      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('"path"', allowedList.length)}` : ''}
-    ) sub
-    GROUP BY "path"
-    ORDER BY score DESC
+      SELECT "path", start_line, end_line, score,
+        row_number() OVER (PARTITION BY "path" ORDER BY score DESC, chunk ASC) AS selected
+      FROM (
+        SELECT "path", chunk, start_line, end_line, ${score} AS score
+        FROM embeddings
+        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('"path"', allowedList.length)}` : ''}
+      ) sub
+    ) ranked
+    WHERE selected = 1
+    ORDER BY score DESC, "path" ASC
     LIMIT ?`;
     const stmt = await duckdb.prepare(sql);
     try {
         const values = [
-            arrayValue(padded(qv, dims)),
+            ...queryNorm === 0 ? [] : [
+                arrayValue(query)
+            ],
             ...allowedList,
             fetch
         ];
         const types = [
-            ARRAY(DOUBLE, dims),
+            ...queryNorm === 0 ? [] : [
+                ARRAY(DOUBLE, dims)
+            ],
             ...allowedList.map(()=>untyped),
             untyped
         ];
@@ -106,14 +117,16 @@ export async function scanSimilar(duckdb, conn, dims, path, opts) {
     const allowedList = opts.allowed ? [
         ...opts.allowed
     ] : [];
-    const sql = `WITH targets(tv) AS (VALUES ${targets.map(()=>'(?)').join(', ')})
-    SELECT e."path" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score
+    const sql = `WITH targets(tv) AS (VALUES ${targets.map(()=>'(?)').join(', ')}), grouped AS (
+    SELECT e."path" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score
     FROM embeddings e, targets t
     WHERE e.vector IS NOT NULL AND e."path" != ?
       ${excludeList.length > 0 ? `AND NOT ${inClause('e."path"', excludeList.length)}` : ''}
       ${opts.allowed ? `AND ${inClause('e."path"', allowedList.length)}` : ''}
     GROUP BY e."path"
-    ORDER BY score DESC
+  )
+    SELECT path, score FROM grouped
+    ORDER BY score DESC, path ASC
     LIMIT ?`;
     const stmt = await duckdb.prepare(sql);
     try {

package/dist/esm/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks the winning chunk's line range from the same row max(score) came from, per path.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","score","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,oIAAoI;AACpI,8QAA8Q;AAC9Q,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,8KAA8K;AAC9K,oGAAoG;AACpG,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;YAAC4B,WAAW7B,OAAO6C,IAAI3C;eAAW+C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ7B;eAAU+C,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC7FoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACjH,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeiB,YAAYlC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAEsB,IAAgE;IACxK,IAAIA,KAAKd,OAAO,IAAIc,KAAKd,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMc,aAAa,MAAMnC,KAAKQ,OAAO,CAAC;IACtC,MAAM4B,aAAc,MAAMD,WAAWE,GAAG,CAACzB;IACzC,IAAIwB,WAAWtD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUtE,aAAaoE,YAAYb,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMqE,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMlB,cAAcY,KAAKd,OAAO,GAAG;WAAIc,KAAKd,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE6B,QAAQf,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEoD,YAAYzD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYwD,YAAYzD,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEoD,KAAKd,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIgE,QAAQf,GAAG,CAAC,CAACkB,IAAMvC,WAAW7B,OAAOoE,GAAGlE;YAASqC;eAAS2B;eAAgBjB;YAAaY,KAAKQ,CAAC;SAAC;QAChI,MAAMhC,QAAsB;eAAI4B,QAAQf,GAAG,CAAC,IAAMpB,MAAMC,QAAQ7B;YAAQJ;eAAYoE,YAAYhB,GAAG,CAAC,IAAMpD;eAAamD,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACxE,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","query","queryNorm","sqrt","reduce","sum","value","score","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,oIAAoI;AACpI,8QAA8Q;AAC9Q,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,mFAAmF;AACnF,oGAAoG;AACpG,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMG,QAAQlD,OAAO6C,IAAI3C;IACzB,MAAMiD,YAAY5C,KAAK6C,IAAI,CAACF,MAAMG,MAAM,CAAC,CAACC,KAAKC,QAAUD,MAAMC,QAAQA,OAAO;IAC9E,MAAMC,QAAQL,cAAc,IAAI,QAAQ;IACxC,MAAMf,MAAM,CAAC;;;;;oDAKqC,EAAEoB,MAAM;;iCAE3B,EAAET,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKvF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAKkD,cAAc,IAAI,EAAE,GAAG;gBAACtB,WAAWqB;aAAO;eAAMD;YAAaH;SAAM;QACtG,MAAMT,QAAsB;eAAKc,cAAc,IAAI,EAAE,GAAG;gBAACrB,MAAMC,QAAQ7B;aAAM;eAAM+C,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC3HoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAEuB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACjH,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeuB,YAAYxC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAE4B,IAAgE;IACxK,IAAIA,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMoB,aAAa,MAAMzC,KAAKQ,OAAO,CAAC;IACtC,MAAMkC,aAAc,MAAMD,WAAWE,GAAG,CAAC/B;IACzC,IAAI8B,WAAW5D,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAM8D,UAAU5E,aAAa0E,YAAYZ,GAAG,CAAC,CAACzC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAM2E,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMxB,cAAckB,KAAKpB,OAAO,GAAG;WAAIoB,KAAKpB,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAEmC,QAAQd,GAAG,CAAC,IAAM,OAAO3C,IAAI,CAAC,MAAM;;;;MAI3E,EAAE0D,YAAY/D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAY8D,YAAY/D,MAAM,GAAG,GAAG,GAAG;MACtF,EAAE0D,KAAKpB,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIsE,QAAQd,GAAG,CAAC,CAACiB,IAAM7C,WAAW7B,OAAO0E,GAAGxE;YAASqC;eAASiC;eAAgBvB;YAAakB,KAAKQ,CAAC;SAAC;QAChI,MAAMtC,QAAsB;eAAIkC,QAAQd,GAAG,CAAC,IAAM3B,MAAMC,QAAQ7B;YAAQJ;eAAY0E,YAAYf,GAAG,CAAC,IAAM3D;eAAamD,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAE0B,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACxE,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF"}

package/dist/esm/store/native.js

@@ -39,13 +39,14 @@ function installOnce(specifier, nodeModulesPath) {
     return promise;
 }
 // Node caches a bare specifier's resolution, including a "not found" miss, for the process
-// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.
-function resolveEntryUrl(importName, nodeModulesPath) {
+// lifetime; use the installed entry directly, as a URL for ESM and a path for CJS.
+function resolveEntrySpecifier(importName, nodeModulesPath) {
     var _pkg_main;
     const pkgDir = join(nodeModulesPath, ...importName.split('/'));
     const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8'));
     const entry = typeof pkg.exports === 'string' ? pkg.exports : (_pkg_main = pkg.main) !== null && _pkg_main !== void 0 ? _pkg_main : 'index.js';
-    return pathToFileURL(join(pkgDir, entry)).href;
+    const entryPath = join(pkgDir, entry);
+    return typeof require === 'undefined' ? pathToFileURL(entryPath).href : entryPath;
 }
 // Import-then-install-then-retry, over an injectable importName so the failure path can be
 // exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.
@@ -60,7 +61,7 @@ export async function loadOrInstall(descriptor, nodeModulesPath, importName = de
             throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" needs ${descriptor.pkg}, and installing it automatically failed (${installErr.message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \`npm install ${descriptor.pkg}\` yourself and try again`);
         }
         try {
-            return await import(resolveEntryUrl(importName, nodeModulesPath));
+            return await import(resolveEntrySpecifier(importName, nodeModulesPath));
         } catch (loadErr) {
             throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" installed ${descriptor.pkg} but it could not be loaded (${loadErr.message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \`npm install ${descriptor.pkg}\` to see the underlying error`);
         }