sensemaking 0.24.3 → 0.24.5

package/dist/esm/store/duckdb/lexical.js

@@ -4,6 +4,7 @@ import { SenseError } from '../../errors.js';
 import { foldForSearch, matchesSearchPhrase, searchTokens } from '../../text/segment.js';
 import { getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
+import { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.js';
 const FIELDS = [
     'title',
     'summary',
@@ -17,7 +18,7 @@ const FIELD_WEIGHT = {
     text: 1
 };
 // FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently
-// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.
+// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.
 const FTS5_OPERATORS = [
     {
         label: 'prefix query',
@@ -38,10 +39,14 @@ const FTS5_OPERATORS = [
     {
         label: 'column filter',
         re: /(?:^|\s)[\p{L}_]\w*\s*:/u
+    },
+    {
+        label: 'grouping operator',
+        re: /[()]/
     }
 ];
 function unsupportedOperator(terms) {
-    const withoutPhrases = terms.replace(/"[^"]*"/g, ' ');
+    const withoutPhrases = terms.replace(/"[^"]*"/g, 'phrase');
     for (const { label, re } of FTS5_OPERATORS){
         const m = withoutPhrases.match(re);
         if (m) return {
@@ -63,7 +68,8 @@ async function stateFor(conn) {
         // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that
         // never built an index, or one a prior process marked stale and never got to clear.
         state = {
-            stale: await getMeta(conn, FTS_STALE_META_KEY) !== '0'
+            stale: await getMeta(conn, FTS_STALE_META_KEY) !== '0',
+            orderedMacroReady: false
         };
         ftsState.set(conn, state);
     }
@@ -73,7 +79,8 @@ async function stateFor(conn) {
 // transaction (reconcile.ts) so a crash never lands the content write without the stale mark.
 export async function markContentStale(conn) {
     ftsState.set(conn, {
-        stale: true
+        stale: true,
+        orderedMacroReady: false
     });
     await setMeta(conn, FTS_STALE_META_KEY, '1');
 }
@@ -126,6 +133,7 @@ function splitTerms(terms) {
 // the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.
 async function ensureFtsFresh(conn, state) {
     if (!state.stale) return;
+    state.orderedMacroReady = false;
     await conn.exec('INSTALL fts; LOAD fts;');
     await withTransaction(conn, async ()=>{
         // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's
@@ -147,16 +155,16 @@ function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
     if (words.length > 0) {
         const wordQuery = words.map(foldForSearch).join(' ');
         for (const field of FIELDS){
-            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
+            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
             scoreParams.push(wordQuery);
         }
         if (bareWords.length > 0) {
-            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const nativeGate = `${ORDERED_BM25_MACRO}(content."path", ?, conjunctive := true) IS NOT NULL`;
             gateParts.push(nativeGate);
             gateParams.push(bareWords.map(foldForSearch).join(' '));
         }
         if (phraseWords.length > 0) {
-            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const nativeGate = `${ORDERED_BM25_MACRO}(content."path", ?, conjunctive := true) IS NOT NULL`;
             const fallback = phraseWords.map((_word)=>`(${FIELDS.map((field)=>`contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');
             gateParts.push(`(${nativeGate} OR (${fallback}))`);
             gateParams.push(phraseWords.map(foldForSearch).join(' '));
@@ -192,6 +200,7 @@ export async function queryLexical(conn, terms, opts, state) {
     const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);
     if (emptyPhrase || words.length === 0 && substrings.length === 0) return [];
     if (words.length > 0) await ensureFtsFresh(conn, state);
+    if (words.length > 0) await ensureOrderedBm25(conn, state);
     const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);
     const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
     const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0' };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","set","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","ensureFtsFresh","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","stmt","prepare","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM1B,QAAQuB,MAAMJ,wBAAyB;QAAI;QACnEC,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeI,iBAAiBL,IAAgB;IACrDH,SAASO,GAAG,CAACJ,MAAM;QAAEG,OAAO;IAAK;IACjC,MAAMzB,QAAQsB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASU,WAAWjB,KAAa;IAC/B,MAAMkB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAMtB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACsB,IAAIC;QACtD,MAAMC,SAASD,MAAMnB,IAAI;QACzB,MAAMqB,eAAexC,aAAauC;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAMtB,SAASsB,aAAc;YAChC,IAAItB,MAAMyB,QAAQ,EAAET,WAAWQ,IAAI,CAACxB,MAAMV,IAAI;iBACzC;gBACHuB,MAAMW,IAAI,CAACxB,MAAMV,IAAI;gBACrByB,YAAYS,IAAI,CAACxB,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMoC,OAAO9B,eAAe+B,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMvB,SAASlB,aAAa4C,KAAM;YACrC,IAAI1B,MAAMyB,QAAQ,EAAET,WAAWQ,IAAI,CAACxB,MAAMV,IAAI;iBACzC;gBACHuB,MAAMW,IAAI,CAACxB,MAAMV,IAAI;gBACrBwB,UAAUU,IAAI,CAACxB,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAEuB;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,eAAetB,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAME,KAAK,EAAE;IAClB,MAAMH,KAAKuB,IAAI,CAAC;IAChB,MAAM5C,gBAAgBqB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKuB,IAAI,CAAC,CAAC,qGAAqG,CAAC;QACvH,0FAA0F;QAC1F,gFAAgF;QAChF,MAAM7C,QAAQsB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASqB,kBAAkBjB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,MAAMe,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMU,MAAM,GAAG,GAAG;QACpB,MAAMY,YAAYtB,MAAMuB,GAAG,CAACxD,eAAeyD,IAAI,CAAC;QAChD,KAAK,MAAMC,SAASpD,OAAQ;YAC1B6C,WAAWP,IAAI,CAAC,GAAGrC,YAAY,CAACmD,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJL,YAAYT,IAAI,CAACW;QACnB;QACA,IAAIrB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMgB,aAAa,CAAC,+EAA+E,CAAC;YACpGP,UAAUR,IAAI,CAACe;YACfL,WAAWV,IAAI,CAACV,UAAUsB,GAAG,CAACxD,eAAeyD,IAAI,CAAC;QACpD;QACA,IAAItB,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMgB,aAAa,CAAC,+EAA+E,CAAC;YACpG,MAAMC,WAAWzB,YAAYqB,GAAG,CAAC,CAACK,QAAU,CAAC,CAAC,EAAEvD,OAAOkD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJL,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEe,WAAW,KAAK,EAAEC,SAAS,EAAE,CAAC;YACjDN,WAAWV,IAAI,CAACT,YAAYqB,GAAG,CAACxD,eAAeyD,IAAI,CAAC;YACpDH,WAAWV,IAAI,IAAIT,YAAY2B,OAAO,CAAC,CAACC,OAASzD,OAAOkD,GAAG,CAAC,IAAMxD,cAAc+D;QAClF;IACF;IAEA,KAAK,MAAMC,OAAO5B,WAAY;QAC5B,MAAM6B,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMR,SAASpD,OAAQ;YAC1B6C,WAAWP,IAAI,CAAC,GAAGrC,YAAY,CAACmD,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHL,YAAYT,IAAI,CAAC5C,cAAciE;QACjC;QACAb,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEtC,OAAOkD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GH,WAAWV,IAAI,IAAItC,OAAOkD,GAAG,CAAC,IAAMxD,cAAciE;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWM,IAAI,CAAC;QAAQW,SAAShB,UAAUK,IAAI,CAAC,YAAY;QAAQY,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAa5C,IAAgB,EAAEX,KAAa,EAAEwD,IAAyB,EAAE5C,KAAoB;IACjH,MAAM6C,cAAc1D,oBAAoBC;IACxC,IAAIyD,gBAAgB,MAAM;QACxB,MAAM,IAAIzE,WAAW,4BAA4B,CAAC,yCAAyC,EAAEyE,YAAY5D,KAAK,CAAC,GAAG,EAAE4D,YAAYpD,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEoD,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAEtC,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWjB;IACvF,IAAIuB,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMK,eAAetB,MAAMC;IAEjD,MAAM,EAAEwC,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGnB,kBAAkBjB,OAAOC,WAAWC,aAAaC;IACvF,MAAMyC,eAAexC,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,WAAWzC,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAMoC,MAAM,CAAC,kBAAkB,EAAE1C,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAEwB,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAME,OAAO,MAAMtD,KAAKuD,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAId,WAAYhC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACiC;KAAM;IAC7E,MAAMQ,UAAU/C,QAAQM,MAAM,KAAK,IAAIuC,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQjD,QAAQkD,KAAK,CAAC,CAAC9C,SAAWnC,OAAOkF,IAAI,CAAC,CAAC9B;oBAA8B4B;uBAApBrF,qBAAoBqF,aAAAA,GAAG,CAAC5B,MAAM,cAAV4B,wBAAAA,aAAc,IAAI7C;;IAC1J,OAAO2C,QAAQK,KAAK,CAAC,GAAGb,OAAOpB,GAAG,CAAC,CAAC,EAAEkC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmBjE,IAAgB;IACjD,OAAO;QACLkE,OAAO,OAAO7E,OAAOwD,OAASD,aAAa5C,MAAMX,OAAOwD,MAAM,MAAM9C,SAASC;QAC7EmE,WAAW,IAAM9D,iBAAiBL;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n  orderedMacroReady: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0', orderedMacroReady: false };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true, orderedMacroReady: false });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  state.orderedMacroReady = false;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n  if (words.length > 0) await ensureOrderedBm25(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","ensureOrderedBm25","ORDERED_BM25_MACRO","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","orderedMacroReady","set","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","ensureFtsFresh","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","stmt","prepare","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAE1E,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAOA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM5B,QAAQyB,MAAMJ,wBAAyB;YAAKQ,mBAAmB;QAAM;QAC7FP,SAASQ,GAAG,CAACL,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeK,iBAAiBN,IAAgB;IACrDH,SAASQ,GAAG,CAACL,MAAM;QAAEG,OAAO;QAAMC,mBAAmB;IAAM;IAC3D,MAAM5B,QAAQwB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASW,WAAWlB,KAAa;IAC/B,MAAMmB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAMvB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACuB,IAAIC;QACtD,MAAMC,SAASD,MAAMpB,IAAI;QACzB,MAAMsB,eAAe3C,aAAa0C;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAMvB,SAASuB,aAAc;YAChC,IAAIvB,MAAM0B,QAAQ,EAAET,WAAWQ,IAAI,CAACzB,MAAMV,IAAI;iBACzC;gBACHwB,MAAMW,IAAI,CAACzB,MAAMV,IAAI;gBACrB0B,YAAYS,IAAI,CAACzB,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMqC,OAAO/B,eAAegC,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMxB,SAASpB,aAAa+C,KAAM;YACrC,IAAI3B,MAAM0B,QAAQ,EAAET,WAAWQ,IAAI,CAACzB,MAAMV,IAAI;iBACzC;gBACHwB,MAAMW,IAAI,CAACzB,MAAMV,IAAI;gBACrByB,UAAUU,IAAI,CAACzB,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAEwB;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,eAAevB,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAME,KAAK,EAAE;IAClBF,MAAMG,iBAAiB,GAAG;IAC1B,MAAMJ,KAAKwB,IAAI,CAAC;IAChB,MAAM/C,gBAAgBuB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKwB,IAAI,CAAC,CAAC,qGAAqG,CAAC;QACvH,0FAA0F;QAC1F,gFAAgF;QAChF,MAAMhD,QAAQwB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASsB,kBAAkBjB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,MAAMe,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMU,MAAM,GAAG,GAAG;QACpB,MAAMY,YAAYtB,MAAMuB,GAAG,CAAC3D,eAAe4D,IAAI,CAAC;QAChD,KAAK,MAAMC,SAASrD,OAAQ;YAC1B8C,WAAWP,IAAI,CAAC,GAAGtC,YAAY,CAACoD,MAAM,CAAC,cAAc,EAAEtD,mBAAmB,+BAA+B,EAAEsD,MAAM,4BAA4B,CAAC;YAC9IL,YAAYT,IAAI,CAACW;QACnB;QACA,IAAIrB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMgB,aAAa,GAAGvD,mBAAmB,oDAAoD,CAAC;YAC9FgD,UAAUR,IAAI,CAACe;YACfL,WAAWV,IAAI,CAACV,UAAUsB,GAAG,CAAC3D,eAAe4D,IAAI,CAAC;QACpD;QACA,IAAItB,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMgB,aAAa,GAAGvD,mBAAmB,oDAAoD,CAAC;YAC9F,MAAMwD,WAAWzB,YAAYqB,GAAG,CAAC,CAACK,QAAU,CAAC,CAAC,EAAExD,OAAOmD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJL,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEe,WAAW,KAAK,EAAEC,SAAS,EAAE,CAAC;YACjDN,WAAWV,IAAI,CAACT,YAAYqB,GAAG,CAAC3D,eAAe4D,IAAI,CAAC;YACpDH,WAAWV,IAAI,IAAIT,YAAY2B,OAAO,CAAC,CAACC,OAAS1D,OAAOmD,GAAG,CAAC,IAAM3D,cAAckE;QAClF;IACF;IAEA,KAAK,MAAMC,OAAO5B,WAAY;QAC5B,MAAM6B,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMR,SAASrD,OAAQ;YAC1B8C,WAAWP,IAAI,CAAC,GAAGtC,YAAY,CAACoD,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHL,YAAYT,IAAI,CAAC/C,cAAcoE;QACjC;QACAb,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEvC,OAAOmD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GH,WAAWV,IAAI,IAAIvC,OAAOmD,GAAG,CAAC,IAAM3D,cAAcoE;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWM,IAAI,CAAC;QAAQW,SAAShB,UAAUK,IAAI,CAAC,YAAY;QAAQY,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAa7C,IAAgB,EAAEX,KAAa,EAAEyD,IAAyB,EAAE7C,KAAoB;IACjH,MAAM8C,cAAc3D,oBAAoBC;IACxC,IAAI0D,gBAAgB,MAAM;QACxB,MAAM,IAAI5E,WAAW,4BAA4B,CAAC,yCAAyC,EAAE4E,YAAY7D,KAAK,CAAC,GAAG,EAAE6D,YAAYrD,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEqD,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAEtC,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWlB;IACvF,IAAIwB,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMK,eAAevB,MAAMC;IACjD,IAAIO,MAAMU,MAAM,GAAG,GAAG,MAAMxC,kBAAkBsB,MAAMC;IAEpD,MAAM,EAAEyC,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGnB,kBAAkBjB,OAAOC,WAAWC,aAAaC;IACvF,MAAMyC,eAAexC,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,WAAWzC,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAMoC,MAAM,CAAC,kBAAkB,EAAE1C,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAEwB,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAME,OAAO,MAAMvD,KAAKwD,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAId,WAAYhC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACiC;KAAM;IAC7E,MAAMQ,UAAU/C,QAAQM,MAAM,KAAK,IAAIuC,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQjD,QAAQkD,KAAK,CAAC,CAAC9C,SAAWpC,OAAOmF,IAAI,CAAC,CAAC9B;oBAA8B4B;uBAApBxF,qBAAoBwF,aAAAA,GAAG,CAAC5B,MAAM,cAAV4B,wBAAAA,aAAc,IAAI7C;;IAC1J,OAAO2C,QAAQK,KAAK,CAAC,GAAGb,OAAOpB,GAAG,CAAC,CAAC,EAAEkC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmBlE,IAAgB;IACjD,OAAO;QACLmE,OAAO,OAAO9E,OAAOyD,OAASD,aAAa7C,MAAMX,OAAOyD,MAAM,MAAM/C,SAASC;QAC7EoE,WAAW,IAAM9D,iBAAiBN;IACpC;AACF"}

package/dist/esm/store/duckdb/native.d.ts

@@ -1,2 +1,4 @@
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
+export declare const DUCKDB_VERSION = "1.5.5-r.4";
+export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.4";
 export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/esm/store/duckdb/native.js

@@ -1,9 +1,13 @@
 import { loadOrInstall, packageNodeModules } from '../native.js';
 export const DUCKDB_PACKAGE = '@duckdb/node-api';
+export const DUCKDB_VERSION = '1.5.5-r.4';
+export const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;
 const DUCKDB = {
     store: 'duckdb',
     pkg: DUCKDB_PACKAGE,
-    sizeHint: '~110MB'
+    sizeHint: '~110MB',
+    version: DUCKDB_VERSION,
+    installSpec: DUCKDB_INSTALL_SPEC
 };
 let duckdbApiPromise;
 // The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning

package/dist/esm/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\nconst DUCKDB: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB' };\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB","store","pkg","sizeHint","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AAEjD,MAAMC,SAA2B;IAAEC,OAAO;IAAUC,KAAKH;IAAgBI,UAAU;AAAS;AAE5F,IAAIC;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBP,cAAiDG,QAAQF;IACnG,OAAOM;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_VERSION = '1.5.5-r.4';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  version: DUCKDB_VERSION,\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB_VERSION","DUCKDB_INSTALL_SPEC","DUCKDB","store","pkg","sizeHint","version","installSpec","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AACjD,OAAO,MAAMC,iBAAiB,YAAY;AAC1C,OAAO,MAAMC,sBAAsB,GAAGF,eAAe,CAAC,EAAEC,gBAAgB,CAAC;AAEzE,MAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKL;IACLM,UAAU;IACVC,SAASN;IACTO,aAAaN;AACf;AAEA,IAAIO;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBX,cAAiDK,QAAQJ;IACnG,OAAOU;AACT"}

package/dist/esm/store/duckdb/ordered-bm25.d.ts

@@ -0,0 +1,17 @@
+import type { Connection } from '../types.js';
+export declare const ORDERED_BM25_MACRO = "sense_ordered_match_bm25";
+interface NativeMacroRow {
+    function_type: string;
+    parameters: unknown;
+    parameter_types: unknown;
+    macro_definition: string;
+}
+interface NativeMacroContract {
+    version: string;
+    macro: NativeMacroRow;
+}
+export declare function validateNativeMacroContract({ version, macro }: NativeMacroContract): string;
+export declare function ensureOrderedBm25(conn: Connection, state: {
+    orderedMacroReady: boolean;
+}): Promise<void>;
+export {};

package/dist/esm/store/duckdb/ordered-bm25.js

@@ -0,0 +1,56 @@
+import { createHash } from 'node:crypto';
+import { SenseError } from '../../errors.js';
+// This is a connection-local TEMP macro. The contract hash covers the native fts definition,
+// DuckDB version, and the reviewed parameter defaults captured in the retained diagnostic.
+export const ORDERED_BM25_MACRO = 'sense_ordered_match_bm25';
+const SUPPORTED_DUCKDB_VERSION = 'v1.5.5';
+const SUPPORTED_CONTRACT_SHA256 = '4476d543e43e7f8c89751cac0659b64e09d906cd3fefb1bfcaf4d15d3b1d9e00';
+const EXPECTED_PARAMETERS = [
+    'docname',
+    'query_string',
+    'fields',
+    'k',
+    'b',
+    'conjunctive'
+];
+const EXPECTED_DEFAULTS = {
+    fields: 'NULL',
+    k: '1.2',
+    b: '0.75',
+    conjunctive: 'false'
+};
+const EXPECTED_SIGNATURE = `docname, query_string, fields := ${EXPECTED_DEFAULTS.fields}, k := ${EXPECTED_DEFAULTS.k}, b := ${EXPECTED_DEFAULTS.b}, conjunctive := ${EXPECTED_DEFAULTS.conjunctive}`;
+const UNORDERED_SUM = 'sum(subscore)';
+function unsupported(message) {
+    return new SenseError('STORE_CAPABILITY_MISSING', `store "duckdb" cannot install its deterministic native BM25 adapter: ${message}; update the DuckDB adapter before using lexical search`);
+}
+function contractSha256(version, definition) {
+    return createHash('sha256').update(`${version}\0${EXPECTED_SIGNATURE}\0${definition}`).digest('hex');
+}
+export function validateNativeMacroContract({ version, macro }) {
+    if (version !== SUPPORTED_DUCKDB_VERSION) throw unsupported(`DuckDB ${version} is unsupported (validated version is ${SUPPORTED_DUCKDB_VERSION})`);
+    if (macro.function_type !== 'macro') throw unsupported(`match_bm25 is ${macro.function_type}, not a macro`);
+    if (JSON.stringify(macro.parameters) !== JSON.stringify(EXPECTED_PARAMETERS)) throw unsupported(`match_bm25 parameters changed (${JSON.stringify(macro.parameters)})`);
+    if (!Array.isArray(macro.parameter_types) || macro.parameter_types.some((type)=>type !== null)) throw unsupported('match_bm25 parameter types changed');
+    if (contractSha256(version, macro.macro_definition) !== SUPPORTED_CONTRACT_SHA256) throw unsupported('match_bm25 definition or reviewed parameter-default contract changed');
+    if (macro.macro_definition.split(UNORDERED_SUM).length !== 2) throw unsupported('match_bm25 does not contain exactly one validated unordered subscore sum');
+    return macro.macro_definition.replace(UNORDERED_SUM, 'sum(subscore ORDER BY subscore)');
+}
+export async function ensureOrderedBm25(conn, state) {
+    if (state.orderedMacroReady) return;
+    const versionStatement = await conn.prepare('SELECT version() AS version');
+    const versionRow = await versionStatement.get();
+    const macroStatement = await conn.prepare(`
+    SELECT function_type, parameters, parameter_types, macro_definition
+    FROM duckdb_functions()
+    WHERE schema_name = 'fts_main_content' AND function_name = 'match_bm25'
+  `);
+    const macro = await macroStatement.get();
+    if (!macro) throw unsupported('fts_main_content.match_bm25 is missing');
+    const orderedDefinition = validateNativeMacroContract({
+        version: versionRow.version,
+        macro
+    });
+    await conn.exec(`CREATE OR REPLACE TEMP MACRO ${ORDERED_BM25_MACRO}(${EXPECTED_SIGNATURE}) AS ${orderedDefinition}`);
+    state.orderedMacroReady = true;
+}

package/dist/esm/store/duckdb/ordered-bm25.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { SenseError } from '../../errors.ts';\nimport type { Connection } from '../types.ts';\n\n// This is a connection-local TEMP macro. The contract hash covers the native fts definition,\n// DuckDB version, and the reviewed parameter defaults captured in the retained diagnostic.\nexport const ORDERED_BM25_MACRO = 'sense_ordered_match_bm25';\nconst SUPPORTED_DUCKDB_VERSION = 'v1.5.5';\nconst SUPPORTED_CONTRACT_SHA256 = '4476d543e43e7f8c89751cac0659b64e09d906cd3fefb1bfcaf4d15d3b1d9e00';\nconst EXPECTED_PARAMETERS = ['docname', 'query_string', 'fields', 'k', 'b', 'conjunctive'];\nconst EXPECTED_DEFAULTS = { fields: 'NULL', k: '1.2', b: '0.75', conjunctive: 'false' } as const;\nconst EXPECTED_SIGNATURE = `docname, query_string, fields := ${EXPECTED_DEFAULTS.fields}, k := ${EXPECTED_DEFAULTS.k}, b := ${EXPECTED_DEFAULTS.b}, conjunctive := ${EXPECTED_DEFAULTS.conjunctive}`;\nconst UNORDERED_SUM = 'sum(subscore)';\n\ninterface NativeMacroRow {\n  function_type: string;\n  parameters: unknown;\n  parameter_types: unknown;\n  macro_definition: string;\n}\n\ninterface NativeMacroContract {\n  version: string;\n  macro: NativeMacroRow;\n}\n\nfunction unsupported(message: string): SenseError {\n  return new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" cannot install its deterministic native BM25 adapter: ${message}; update the DuckDB adapter before using lexical search`);\n}\n\nfunction contractSha256(version: string, definition: string): string {\n  return createHash('sha256').update(`${version}\\0${EXPECTED_SIGNATURE}\\0${definition}`).digest('hex');\n}\n\nexport function validateNativeMacroContract({ version, macro }: NativeMacroContract): string {\n  if (version !== SUPPORTED_DUCKDB_VERSION) throw unsupported(`DuckDB ${version} is unsupported (validated version is ${SUPPORTED_DUCKDB_VERSION})`);\n  if (macro.function_type !== 'macro') throw unsupported(`match_bm25 is ${macro.function_type}, not a macro`);\n  if (JSON.stringify(macro.parameters) !== JSON.stringify(EXPECTED_PARAMETERS)) throw unsupported(`match_bm25 parameters changed (${JSON.stringify(macro.parameters)})`);\n  if (!Array.isArray(macro.parameter_types) || macro.parameter_types.some((type) => type !== null)) throw unsupported('match_bm25 parameter types changed');\n  if (contractSha256(version, macro.macro_definition) !== SUPPORTED_CONTRACT_SHA256) throw unsupported('match_bm25 definition or reviewed parameter-default contract changed');\n  if (macro.macro_definition.split(UNORDERED_SUM).length !== 2) throw unsupported('match_bm25 does not contain exactly one validated unordered subscore sum');\n  return macro.macro_definition.replace(UNORDERED_SUM, 'sum(subscore ORDER BY subscore)');\n}\n\nexport async function ensureOrderedBm25(conn: Connection, state: { orderedMacroReady: boolean }): Promise<void> {\n  if (state.orderedMacroReady) return;\n  const versionStatement = await conn.prepare('SELECT version() AS version');\n  const versionRow = (await versionStatement.get()) as { version: string };\n  const macroStatement = await conn.prepare(`\n    SELECT function_type, parameters, parameter_types, macro_definition\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content' AND function_name = 'match_bm25'\n  `);\n  const macro = (await macroStatement.get()) as NativeMacroRow | undefined;\n  if (!macro) throw unsupported('fts_main_content.match_bm25 is missing');\n  const orderedDefinition = validateNativeMacroContract({ version: versionRow.version, macro });\n  await conn.exec(`CREATE OR REPLACE TEMP MACRO ${ORDERED_BM25_MACRO}(${EXPECTED_SIGNATURE}) AS ${orderedDefinition}`);\n  state.orderedMacroReady = true;\n}\n"],"names":["createHash","SenseError","ORDERED_BM25_MACRO","SUPPORTED_DUCKDB_VERSION","SUPPORTED_CONTRACT_SHA256","EXPECTED_PARAMETERS","EXPECTED_DEFAULTS","fields","k","b","conjunctive","EXPECTED_SIGNATURE","UNORDERED_SUM","unsupported","message","contractSha256","version","definition","update","digest","validateNativeMacroContract","macro","function_type","JSON","stringify","parameters","Array","isArray","parameter_types","some","type","macro_definition","split","length","replace","ensureOrderedBm25","conn","state","orderedMacroReady","versionStatement","prepare","versionRow","get","macroStatement","orderedDefinition","exec"],"mappings":"AAAA,SAASA,UAAU,QAAQ,cAAc;AACzC,SAASC,UAAU,QAAQ,kBAAkB;AAG7C,6FAA6F;AAC7F,2FAA2F;AAC3F,OAAO,MAAMC,qBAAqB,2BAA2B;AAC7D,MAAMC,2BAA2B;AACjC,MAAMC,4BAA4B;AAClC,MAAMC,sBAAsB;IAAC;IAAW;IAAgB;IAAU;IAAK;IAAK;CAAc;AAC1F,MAAMC,oBAAoB;IAAEC,QAAQ;IAAQC,GAAG;IAAOC,GAAG;IAAQC,aAAa;AAAQ;AACtF,MAAMC,qBAAqB,CAAC,iCAAiC,EAAEL,kBAAkBC,MAAM,CAAC,OAAO,EAAED,kBAAkBE,CAAC,CAAC,OAAO,EAAEF,kBAAkBG,CAAC,CAAC,iBAAiB,EAAEH,kBAAkBI,WAAW,EAAE;AACpM,MAAME,gBAAgB;AActB,SAASC,YAAYC,OAAe;IAClC,OAAO,IAAIb,WAAW,4BAA4B,CAAC,qEAAqE,EAAEa,QAAQ,uDAAuD,CAAC;AAC5L;AAEA,SAASC,eAAeC,OAAe,EAAEC,UAAkB;IACzD,OAAOjB,WAAW,UAAUkB,MAAM,CAAC,GAAGF,QAAQ,EAAE,EAAEL,mBAAmB,EAAE,EAAEM,YAAY,EAAEE,MAAM,CAAC;AAChG;AAEA,OAAO,SAASC,4BAA4B,EAAEJ,OAAO,EAAEK,KAAK,EAAuB;IACjF,IAAIL,YAAYb,0BAA0B,MAAMU,YAAY,CAAC,OAAO,EAAEG,QAAQ,sCAAsC,EAAEb,yBAAyB,CAAC,CAAC;IACjJ,IAAIkB,MAAMC,aAAa,KAAK,SAAS,MAAMT,YAAY,CAAC,cAAc,EAAEQ,MAAMC,aAAa,CAAC,aAAa,CAAC;IAC1G,IAAIC,KAAKC,SAAS,CAACH,MAAMI,UAAU,MAAMF,KAAKC,SAAS,CAACnB,sBAAsB,MAAMQ,YAAY,CAAC,+BAA+B,EAAEU,KAAKC,SAAS,CAACH,MAAMI,UAAU,EAAE,CAAC,CAAC;IACrK,IAAI,CAACC,MAAMC,OAAO,CAACN,MAAMO,eAAe,KAAKP,MAAMO,eAAe,CAACC,IAAI,CAAC,CAACC,OAASA,SAAS,OAAO,MAAMjB,YAAY;IACpH,IAAIE,eAAeC,SAASK,MAAMU,gBAAgB,MAAM3B,2BAA2B,MAAMS,YAAY;IACrG,IAAIQ,MAAMU,gBAAgB,CAACC,KAAK,CAACpB,eAAeqB,MAAM,KAAK,GAAG,MAAMpB,YAAY;IAChF,OAAOQ,MAAMU,gBAAgB,CAACG,OAAO,CAACtB,eAAe;AACvD;AAEA,OAAO,eAAeuB,kBAAkBC,IAAgB,EAAEC,KAAqC;IAC7F,IAAIA,MAAMC,iBAAiB,EAAE;IAC7B,MAAMC,mBAAmB,MAAMH,KAAKI,OAAO,CAAC;IAC5C,MAAMC,aAAc,MAAMF,iBAAiBG,GAAG;IAC9C,MAAMC,iBAAiB,MAAMP,KAAKI,OAAO,CAAC,CAAC;;;;EAI3C,CAAC;IACD,MAAMnB,QAAS,MAAMsB,eAAeD,GAAG;IACvC,IAAI,CAACrB,OAAO,MAAMR,YAAY;IAC9B,MAAM+B,oBAAoBxB,4BAA4B;QAAEJ,SAASyB,WAAWzB,OAAO;QAAEK;IAAM;IAC3F,MAAMe,KAAKS,IAAI,CAAC,CAAC,6BAA6B,EAAE3C,mBAAmB,CAAC,EAAES,mBAAmB,KAAK,EAAEiC,mBAAmB;IACnHP,MAAMC,iBAAiB,GAAG;AAC5B"}

package/dist/esm/store/lock-wait.d.ts

@@ -1,2 +1,2 @@
-export declare function recordLockWaitMs(baseDir: string, ms: number): void;
-export declare function lockWaitBudgetMs(baseDir: string): number;
+export declare function recordLockWaitMs(configDir: string, ms: number): void;
+export declare function lockWaitBudgetMs(configDir: string): number;

package/dist/esm/store/lock-wait.js

@@ -6,31 +6,34 @@ import { STATE_DIR } from '../config/index.js';
 const SIDECAR_FILENAME = 'lock-wait.json';
 const FLOOR_MS = 5000;
 const CEILING_MS = 600000;
-function sidecarPath(baseDir) {
-    return join(baseDir, STATE_DIR, SIDECAR_FILENAME);
+function sidecarPath(configDir) {
+    return join(configDir, STATE_DIR, SIDECAR_FILENAME);
 }
-function readRecordedMs(baseDir) {
+function readRecordedMs(configDir) {
     try {
-        const raw = JSON.parse(readFileSync(sidecarPath(baseDir), 'utf8'));
-        return typeof raw.reconcile_max_ms === 'number' ? raw.reconcile_max_ms : 0;
+        const raw = JSON.parse(readFileSync(sidecarPath(configDir), 'utf8'));
+        const value = raw.reconcile_max_ms;
+        return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined;
     } catch  {
-        return 0;
+        return undefined;
     }
 }
-// Called after every reconcile's write transaction, every store alike, so a fresh duckdb/turso
-// tree starts building this the first time anything writes.
-export function recordLockWaitMs(baseDir, ms) {
-    if (ms <= readRecordedMs(baseDir)) return;
+// Called after every reconcile's write transaction, every store alike, so a fresh cache records
+// its first write even when the measured duration is zero.
+export function recordLockWaitMs(configDir, ms) {
+    const recordedMs = readRecordedMs(configDir);
+    if (recordedMs !== undefined && ms <= recordedMs) return;
     try {
-        writeFileSync(sidecarPath(baseDir), JSON.stringify({
+        writeFileSync(sidecarPath(configDir), JSON.stringify({
             reconcile_max_ms: ms
         }));
     } catch  {
     // Best-effort: a write failure here only narrows the next waiter's budget back to the floor.
     }
 }
-// 3x the largest reconcile this tree has recorded, floored and capped -- a waiter outlasts a
+// 3x the largest reconcile this cache has recorded, floored and capped -- a waiter outlasts a
 // real reconcile and still fails behind a hung one.
-export function lockWaitBudgetMs(baseDir) {
-    return Math.min(Math.max(FLOOR_MS, 3 * readRecordedMs(baseDir)), CEILING_MS);
+export function lockWaitBudgetMs(configDir) {
+    var _readRecordedMs;
+    return Math.min(Math.max(FLOOR_MS, 3 * ((_readRecordedMs = readRecordedMs(configDir)) !== null && _readRecordedMs !== void 0 ? _readRecordedMs : 0)), CEILING_MS);
 }

package/dist/esm/store/lock-wait.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/lock-wait.ts"],"sourcesContent":["import { readFileSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { STATE_DIR } from '../config/index.ts';\n\n// duckdb/turso lock the cache file exclusively, so a waiter can't connect and read reconcile_max_ms\n// from meta; this sidecar is plain fs, readable while locked. Same derivation as busy_timeout (open.ts).\nconst SIDECAR_FILENAME = 'lock-wait.json';\nconst FLOOR_MS = 5_000;\nconst CEILING_MS = 600_000;\n\nfunction sidecarPath(baseDir: string): string {\n  return join(baseDir, STATE_DIR, SIDECAR_FILENAME);\n}\n\nfunction readRecordedMs(baseDir: string): number {\n  try {\n    const raw = JSON.parse(readFileSync(sidecarPath(baseDir), 'utf8'));\n    return typeof raw.reconcile_max_ms === 'number' ? raw.reconcile_max_ms : 0;\n  } catch {\n    return 0;\n  }\n}\n\n// Called after every reconcile's write transaction, every store alike, so a fresh duckdb/turso\n// tree starts building this the first time anything writes.\nexport function recordLockWaitMs(baseDir: string, ms: number): void {\n  if (ms <= readRecordedMs(baseDir)) return;\n  try {\n    writeFileSync(sidecarPath(baseDir), JSON.stringify({ reconcile_max_ms: ms }));\n  } catch {\n    // Best-effort: a write failure here only narrows the next waiter's budget back to the floor.\n  }\n}\n\n// 3x the largest reconcile this tree has recorded, floored and capped -- a waiter outlasts a\n// real reconcile and still fails behind a hung one.\nexport function lockWaitBudgetMs(baseDir: string): number {\n  return Math.min(Math.max(FLOOR_MS, 3 * readRecordedMs(baseDir)), CEILING_MS);\n}\n"],"names":["readFileSync","writeFileSync","join","STATE_DIR","SIDECAR_FILENAME","FLOOR_MS","CEILING_MS","sidecarPath","baseDir","readRecordedMs","raw","JSON","parse","reconcile_max_ms","recordLockWaitMs","ms","stringify","lockWaitBudgetMs","Math","min","max"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,QAAQ,UAAU;AACtD,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,oGAAoG;AACpG,yGAAyG;AACzG,MAAMC,mBAAmB;AACzB,MAAMC,WAAW;AACjB,MAAMC,aAAa;AAEnB,SAASC,YAAYC,OAAe;IAClC,OAAON,KAAKM,SAASL,WAAWC;AAClC;AAEA,SAASK,eAAeD,OAAe;IACrC,IAAI;QACF,MAAME,MAAMC,KAAKC,KAAK,CAACZ,aAAaO,YAAYC,UAAU;QAC1D,OAAO,OAAOE,IAAIG,gBAAgB,KAAK,WAAWH,IAAIG,gBAAgB,GAAG;IAC3E,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,+FAA+F;AAC/F,4DAA4D;AAC5D,OAAO,SAASC,iBAAiBN,OAAe,EAAEO,EAAU;IAC1D,IAAIA,MAAMN,eAAeD,UAAU;IACnC,IAAI;QACFP,cAAcM,YAAYC,UAAUG,KAAKK,SAAS,CAAC;YAAEH,kBAAkBE;QAAG;IAC5E,EAAE,OAAM;IACN,6FAA6F;IAC/F;AACF;AAEA,6FAA6F;AAC7F,oDAAoD;AACpD,OAAO,SAASE,iBAAiBT,OAAe;IAC9C,OAAOU,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAACf,UAAU,IAAII,eAAeD,WAAWF;AACnE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/lock-wait.ts"],"sourcesContent":["import { readFileSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { STATE_DIR } from '../config/index.ts';\n\n// duckdb/turso lock the cache file exclusively, so a waiter can't connect and read reconcile_max_ms\n// from meta; this sidecar is plain fs, readable while locked. Same derivation as busy_timeout (open.ts).\nconst SIDECAR_FILENAME = 'lock-wait.json';\nconst FLOOR_MS = 5_000;\nconst CEILING_MS = 600_000;\n\nfunction sidecarPath(configDir: string): string {\n  return join(configDir, STATE_DIR, SIDECAR_FILENAME);\n}\n\nfunction readRecordedMs(configDir: string): number | undefined {\n  try {\n    const raw = JSON.parse(readFileSync(sidecarPath(configDir), 'utf8'));\n    const value = raw.reconcile_max_ms;\n    return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\n// Called after every reconcile's write transaction, every store alike, so a fresh cache records\n// its first write even when the measured duration is zero.\nexport function recordLockWaitMs(configDir: string, ms: number): void {\n  const recordedMs = readRecordedMs(configDir);\n  if (recordedMs !== undefined && ms <= recordedMs) return;\n  try {\n    writeFileSync(sidecarPath(configDir), JSON.stringify({ reconcile_max_ms: ms }));\n  } catch {\n    // Best-effort: a write failure here only narrows the next waiter's budget back to the floor.\n  }\n}\n\n// 3x the largest reconcile this cache has recorded, floored and capped -- a waiter outlasts a\n// real reconcile and still fails behind a hung one.\nexport function lockWaitBudgetMs(configDir: string): number {\n  return Math.min(Math.max(FLOOR_MS, 3 * (readRecordedMs(configDir) ?? 0)), CEILING_MS);\n}\n"],"names":["readFileSync","writeFileSync","join","STATE_DIR","SIDECAR_FILENAME","FLOOR_MS","CEILING_MS","sidecarPath","configDir","readRecordedMs","raw","JSON","parse","value","reconcile_max_ms","Number","isFinite","undefined","recordLockWaitMs","ms","recordedMs","stringify","lockWaitBudgetMs","Math","min","max"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,QAAQ,UAAU;AACtD,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,oGAAoG;AACpG,yGAAyG;AACzG,MAAMC,mBAAmB;AACzB,MAAMC,WAAW;AACjB,MAAMC,aAAa;AAEnB,SAASC,YAAYC,SAAiB;IACpC,OAAON,KAAKM,WAAWL,WAAWC;AACpC;AAEA,SAASK,eAAeD,SAAiB;IACvC,IAAI;QACF,MAAME,MAAMC,KAAKC,KAAK,CAACZ,aAAaO,YAAYC,YAAY;QAC5D,MAAMK,QAAQH,IAAII,gBAAgB;QAClC,OAAO,OAAOD,UAAU,YAAYE,OAAOC,QAAQ,CAACH,UAAUA,SAAS,IAAIA,QAAQI;IACrF,EAAE,OAAM;QACN,OAAOA;IACT;AACF;AAEA,gGAAgG;AAChG,2DAA2D;AAC3D,OAAO,SAASC,iBAAiBV,SAAiB,EAAEW,EAAU;IAC5D,MAAMC,aAAaX,eAAeD;IAClC,IAAIY,eAAeH,aAAaE,MAAMC,YAAY;IAClD,IAAI;QACFnB,cAAcM,YAAYC,YAAYG,KAAKU,SAAS,CAAC;YAAEP,kBAAkBK;QAAG;IAC9E,EAAE,OAAM;IACN,6FAA6F;IAC/F;AACF;AAEA,8FAA8F;AAC9F,oDAAoD;AACpD,OAAO,SAASG,iBAAiBd,SAAiB;QACRC;IAAxC,OAAOc,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAACpB,UAAU,MAAKI,kBAAAA,eAAeD,wBAAfC,6BAAAA,kBAA6B,KAAKH;AAC5E"}

package/dist/esm/store/native.d.ts

@@ -2,6 +2,10 @@ export interface NativeDescriptor {
     store: string;
     pkg: string;
     sizeHint: string;
+    /** Optional exact package version required by this native binding. */
+    version?: string;
+    /** npm specifier used only when installation is required (imports stay bare). */
+    installSpec?: string;
 }
 export declare function packageNodeModules(): string;
 export declare function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName?: string): Promise<T>;

package/dist/esm/store/native.js

@@ -48,22 +48,76 @@ function resolveEntrySpecifier(importName, nodeModulesPath) {
     const entryPath = join(pkgDir, entry);
     return typeof require === 'undefined' ? pathToFileURL(entryPath).href : entryPath;
 }
+function readPackageMetadata(pkgDir) {
+    try {
+        return JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8'));
+    } catch  {
+        return undefined;
+    }
+}
+function targetPackageMetadata(importName, nodeModulesPath) {
+    const pkgDir = join(nodeModulesPath, ...importName.split('/'));
+    return {
+        exists: existsSync(pkgDir),
+        metadata: readPackageMetadata(pkgDir)
+    };
+}
+function resolvedPackageMetadata(importName) {
+    try {
+        let dir = dirname(_require.resolve(importName));
+        for(;;){
+            const manifest = join(dir, 'package.json');
+            if (existsSync(manifest)) {
+                const pkg = readPackageMetadata(dir);
+                if (!pkg) return undefined;
+                if (pkg.name === importName) return pkg;
+            }
+            const parent = dirname(dir);
+            if (parent === dir) return undefined;
+            dir = parent;
+        }
+    } catch  {
+        return undefined;
+    }
+}
+function versionContractError(descriptor, installed) {
+    var _descriptor_installSpec;
+    const installSpec = (_descriptor_installSpec = descriptor.installSpec) !== null && _descriptor_installSpec !== void 0 ? _descriptor_installSpec : descriptor.pkg;
+    const detail = installed ? `${descriptor.pkg}@${installed} is installed` : `${descriptor.pkg} has missing or unreadable version metadata`;
+    return new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" needs ${installSpec}, but ${detail}; refusing to overwrite it -- run \`npm install ${installSpec}\` yourself and try again`);
+}
+function assertExpectedVersion(descriptor, metadata) {
+    if (!descriptor.version) return;
+    if (!(metadata === null || metadata === void 0 ? void 0 : metadata.version) || metadata.version !== descriptor.version) throw versionContractError(descriptor, metadata === null || metadata === void 0 ? void 0 : metadata.version);
+}
 // Import-then-install-then-retry, over an injectable importName so the failure path can be
 // exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.
 export async function loadOrInstall(descriptor, nodeModulesPath, importName = descriptor.pkg) {
+    if (descriptor.version) {
+        const target = targetPackageMetadata(importName, nodeModulesPath);
+        if (target.exists) assertExpectedVersion(descriptor, target.metadata);
+    }
+    let loaded;
     try {
-        return await import(importName);
+        loaded = await import(importName);
     } catch  {
-        console.error(`sense: store "${descriptor.store}" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);
+        var _descriptor_installSpec, _descriptor_installSpec1;
+        const installSpec = (_descriptor_installSpec = descriptor.installSpec) !== null && _descriptor_installSpec !== void 0 ? _descriptor_installSpec : importName;
+        const displaySpec = (_descriptor_installSpec1 = descriptor.installSpec) !== null && _descriptor_installSpec1 !== void 0 ? _descriptor_installSpec1 : descriptor.pkg;
+        console.error(`sense: store "${descriptor.store}" needs ${displaySpec}; installing it now (one-time download, ${descriptor.sizeHint})...`);
         try {
-            await installOnce(importName, nodeModulesPath);
+            await installOnce(installSpec, nodeModulesPath);
         } catch (installErr) {
-            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" needs ${descriptor.pkg}, and installing it automatically failed (${installErr.message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \`npm install ${descriptor.pkg}\` yourself and try again`);
+            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" needs ${displaySpec}, and installing it automatically failed (${installErr.message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \`npm install ${displaySpec}\` yourself and try again`);
         }
         try {
-            return await import(resolveEntrySpecifier(importName, nodeModulesPath));
+            const loaded = await import(resolveEntrySpecifier(importName, nodeModulesPath));
+            assertExpectedVersion(descriptor, targetPackageMetadata(importName, nodeModulesPath).metadata);
+            return loaded;
         } catch (loadErr) {
-            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" installed ${descriptor.pkg} but it could not be loaded (${loadErr.message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \`npm install ${descriptor.pkg}\` to see the underlying error`);
+            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" installed ${displaySpec} but it could not be loaded (${loadErr.message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \`npm install ${displaySpec}\` to see the underlying error`);
         }
     }
+    if (descriptor.version) assertExpectedVersion(descriptor, resolvedPackageMetadata(importName));
+    return loaded;
 }

package/dist/esm/store/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; use the installed entry directly, as a URL for ESM and a path for CJS.\nfunction resolveEntrySpecifier(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  const entryPath = join(pkgDir, entry);\n  return typeof require === 'undefined' ? pathToFileURL(entryPath).href : entryPath;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntrySpecifier(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntrySpecifier","importName","pkgDir","split","entry","exports","main","entryPath","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,+FAA+F;AAC/F,sGAAsG;AACtG,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,sFAAsF;AACtF,2EAA2E;AAC3E,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,mFAAmF;AACnF,SAASQ,sBAAsBC,UAAkB,EAAEX,eAAuB;QAGTV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,MAAM2B,YAAYxC,KAAKmC,QAAQE;IAC/B,OAAO,OAAOhC,YAAY,cAAcH,cAAcsC,WAAWC,IAAI,GAAGD;AAC1E;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,OAAO,eAAeE,cAAiBC,UAA4B,EAAEpB,eAAuB,EAAEW,aAAqBS,WAAW9B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNU,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW9B,GAAG,CAAC,wCAAwC,EAAE8B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAM1B,YAAYa,YAAYX;QAChC,EAAE,OAAOyB,YAAY;YACnB,MAAM,IAAI7C,WACR,4BACA,CAAC,OAAO,EAAEwC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW9B,GAAG,CAAC,0CAA0C,EAAE,AAACmC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW9B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,sBAAsBC,YAAYX;QACzD,EAAE,OAAO2B,SAAS;YAChB,MAAM,IAAI/C,WACR,4BACA,CAAC,OAAO,EAAEwC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW9B,GAAG,CAAC,6BAA6B,EAAE,AAACqC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW9B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n  /** Optional exact package version required by this native binding. */\n  version?: string;\n  /** npm specifier used only when installation is required (imports stay bare). */\n  installSpec?: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; use the installed entry directly, as a URL for ESM and a path for CJS.\nfunction resolveEntrySpecifier(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  const entryPath = join(pkgDir, entry);\n  return typeof require === 'undefined' ? pathToFileURL(entryPath).href : entryPath;\n}\n\ntype PackageMetadata = { name?: string; version?: string };\n\nfunction readPackageMetadata(pkgDir: string): PackageMetadata | undefined {\n  try {\n    return JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as PackageMetadata;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction targetPackageMetadata(importName: string, nodeModulesPath: string): { exists: boolean; metadata?: PackageMetadata } {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  return { exists: existsSync(pkgDir), metadata: readPackageMetadata(pkgDir) };\n}\n\nfunction resolvedPackageMetadata(importName: string): PackageMetadata | undefined {\n  try {\n    let dir = dirname(_require.resolve(importName));\n    for (;;) {\n      const manifest = join(dir, 'package.json');\n      if (existsSync(manifest)) {\n        const pkg = readPackageMetadata(dir);\n        if (!pkg) return undefined;\n        if (pkg.name === importName) return pkg;\n      }\n      const parent = dirname(dir);\n      if (parent === dir) return undefined;\n      dir = parent;\n    }\n  } catch {\n    return undefined;\n  }\n}\n\nfunction versionContractError(descriptor: NativeDescriptor, installed?: string): SenseError {\n  const installSpec = descriptor.installSpec ?? descriptor.pkg;\n  const detail = installed ? `${descriptor.pkg}@${installed} is installed` : `${descriptor.pkg} has missing or unreadable version metadata`;\n  return new SenseError('STORE_DEPENDENCY_MISSING', `store \"${descriptor.store}\" needs ${installSpec}, but ${detail}; refusing to overwrite it -- run \\`npm install ${installSpec}\\` yourself and try again`);\n}\n\nfunction assertExpectedVersion(descriptor: NativeDescriptor, metadata: PackageMetadata | undefined): void {\n  if (!descriptor.version) return;\n  if (!metadata?.version || metadata.version !== descriptor.version) throw versionContractError(descriptor, metadata?.version);\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  if (descriptor.version) {\n    const target = targetPackageMetadata(importName, nodeModulesPath);\n    if (target.exists) assertExpectedVersion(descriptor, target.metadata);\n  }\n  let loaded: T;\n  try {\n    loaded = (await import(importName)) as T;\n  } catch {\n    const installSpec = descriptor.installSpec ?? importName;\n    const displaySpec = descriptor.installSpec ?? descriptor.pkg;\n    console.error(`sense: store \"${descriptor.store}\" needs ${displaySpec}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(installSpec, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${displaySpec}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${displaySpec}\\` yourself and try again`\n      );\n    }\n    try {\n      const loaded = (await import(resolveEntrySpecifier(importName, nodeModulesPath))) as T;\n      assertExpectedVersion(descriptor, targetPackageMetadata(importName, nodeModulesPath).metadata);\n      return loaded;\n    } catch (loadErr) {\n      throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"${descriptor.store}\" installed ${displaySpec} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${displaySpec}\\` to see the underlying error`);\n    }\n  }\n  if (descriptor.version) assertExpectedVersion(descriptor, resolvedPackageMetadata(importName));\n  return loaded;\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntrySpecifier","importName","pkgDir","split","entry","exports","main","entryPath","href","readPackageMetadata","undefined","targetPackageMetadata","exists","metadata","resolvedPackageMetadata","resolve","manifest","versionContractError","descriptor","installed","installSpec","detail","store","assertExpectedVersion","version","loadOrInstall","target","loaded","displaySpec","console","error","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAa1C,+FAA+F;AAC/F,sGAAsG;AACtG,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,sFAAsF;AACtF,2EAA2E;AAC3E,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,mFAAmF;AACnF,SAASQ,sBAAsBC,UAAkB,EAAEX,eAAuB;QAGTV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,MAAM2B,YAAYxC,KAAKmC,QAAQE;IAC/B,OAAO,OAAOhC,YAAY,cAAcH,cAAcsC,WAAWC,IAAI,GAAGD;AAC1E;AAIA,SAASE,oBAAoBP,MAAc;IACzC,IAAI;QACF,OAAOrB,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAC/D,EAAE,OAAM;QACN,OAAOQ;IACT;AACF;AAEA,SAASC,sBAAsBV,UAAkB,EAAEX,eAAuB;IACxE,MAAMY,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,OAAO;QAAES,QAAQjD,WAAWuC;QAASW,UAAUJ,oBAAoBP;IAAQ;AAC7E;AAEA,SAASY,wBAAwBb,UAAkB;IACjD,IAAI;QACF,IAAIvB,MAAMZ,QAAQK,SAAS4C,OAAO,CAACd;QACnC,OAAS;YACP,MAAMe,WAAWjD,KAAKW,KAAK;YAC3B,IAAIf,WAAWqD,WAAW;gBACxB,MAAMpC,MAAM6B,oBAAoB/B;gBAChC,IAAI,CAACE,KAAK,OAAO8B;gBACjB,IAAI9B,IAAIG,IAAI,KAAKkB,YAAY,OAAOrB;YACtC;YACA,MAAMI,SAASlB,QAAQY;YACvB,IAAIM,WAAWN,KAAK,OAAOgC;YAC3BhC,MAAMM;QACR;IACF,EAAE,OAAM;QACN,OAAO0B;IACT;AACF;AAEA,SAASO,qBAAqBC,UAA4B,EAAEC,SAAkB;QACxDD;IAApB,MAAME,eAAcF,0BAAAA,WAAWE,WAAW,cAAtBF,qCAAAA,0BAA0BA,WAAWtC,GAAG;IAC5D,MAAMyC,SAASF,YAAY,GAAGD,WAAWtC,GAAG,CAAC,CAAC,EAAEuC,UAAU,aAAa,CAAC,GAAG,GAAGD,WAAWtC,GAAG,CAAC,2CAA2C,CAAC;IACzI,OAAO,IAAIV,WAAW,4BAA4B,CAAC,OAAO,EAAEgD,WAAWI,KAAK,CAAC,QAAQ,EAAEF,YAAY,MAAM,EAAEC,OAAO,gDAAgD,EAAED,YAAY,yBAAyB,CAAC;AAC5M;AAEA,SAASG,sBAAsBL,UAA4B,EAAEL,QAAqC;IAChG,IAAI,CAACK,WAAWM,OAAO,EAAE;IACzB,IAAI,EAACX,qBAAAA,+BAAAA,SAAUW,OAAO,KAAIX,SAASW,OAAO,KAAKN,WAAWM,OAAO,EAAE,MAAMP,qBAAqBC,YAAYL,qBAAAA,+BAAAA,SAAUW,OAAO;AAC7H;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,OAAO,eAAeC,cAAiBP,UAA4B,EAAE5B,eAAuB,EAAEW,aAAqBiB,WAAWtC,GAAG;IAC/H,IAAIsC,WAAWM,OAAO,EAAE;QACtB,MAAME,SAASf,sBAAsBV,YAAYX;QACjD,IAAIoC,OAAOd,MAAM,EAAEW,sBAAsBL,YAAYQ,OAAOb,QAAQ;IACtE;IACA,IAAIc;IACJ,IAAI;QACFA,SAAU,MAAM,MAAM,CAAC1B;IACzB,EAAE,OAAM;YACciB,yBACAA;QADpB,MAAME,eAAcF,0BAAAA,WAAWE,WAAW,cAAtBF,qCAAAA,0BAA0BjB;QAC9C,MAAM2B,eAAcV,2BAAAA,WAAWE,WAAW,cAAtBF,sCAAAA,2BAA0BA,WAAWtC,GAAG;QAC5DiD,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEZ,WAAWI,KAAK,CAAC,QAAQ,EAAEM,YAAY,wCAAwC,EAAEV,WAAWa,QAAQ,CAAC,IAAI,CAAC;QACzI,IAAI;YACF,MAAM3C,YAAYgC,aAAa9B;QACjC,EAAE,OAAO0C,YAAY;YACnB,MAAM,IAAI9D,WACR,4BACA,CAAC,OAAO,EAAEgD,WAAWI,KAAK,CAAC,QAAQ,EAAEM,YAAY,0CAA0C,EAAE,AAACI,WAAqBC,OAAO,CAAC,sKAAsK,EAAEL,YAAY,yBAAyB,CAAC;QAE7U;QACA,IAAI;YACF,MAAMD,SAAU,MAAM,MAAM,CAAC3B,sBAAsBC,YAAYX;YAC/DiC,sBAAsBL,YAAYP,sBAAsBV,YAAYX,iBAAiBuB,QAAQ;YAC7F,OAAOc;QACT,EAAE,OAAOO,SAAS;YAChB,MAAM,IAAIhE,WAAW,4BAA4B,CAAC,OAAO,EAAEgD,WAAWI,KAAK,CAAC,YAAY,EAAEM,YAAY,6BAA6B,EAAE,AAACM,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAET,YAAY,8BAA8B,CAAC;QACvT;IACF;IACA,IAAIV,WAAWM,OAAO,EAAED,sBAAsBL,YAAYJ,wBAAwBb;IAClF,OAAO0B;AACT"}

package/dist/esm/store/open.js

@@ -17,9 +17,8 @@ import { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './sig
 // Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.
 const LOCK_POLL_MS = 50;
 const lockWait = channel('sensemaking.store.lock-wait');
-async function connectUnlocked(dbPath, cfg, dialect) {
-    var _cfg_configDir;
-    const budgetMs = lockWaitBudgetMs((_cfg_configDir = cfg.configDir) !== null && _cfg_configDir !== void 0 ? _cfg_configDir : cfg.baseDir);
+async function connectUnlocked(dbPath, cfg, configDir, dialect) {
+    const budgetMs = lockWaitBudgetMs(configDir);
     const deadline = Date.now() + budgetMs;
     for(;;){
         try {
@@ -41,18 +40,22 @@ async function connectUnlocked(dbPath, cfg, dialect) {
 }
 async function connectWithDialect(cfg, dialect) {
     var _cfg_configDir, _cfg_rootDir;
-    const stateDir = join((_cfg_configDir = cfg.configDir) !== null && _cfg_configDir !== void 0 ? _cfg_configDir : cfg.baseDir, STATE_DIR);
+    const configDir = (_cfg_configDir = cfg.configDir) !== null && _cfg_configDir !== void 0 ? _cfg_configDir : cfg.baseDir;
+    const stateDir = join(configDir, STATE_DIR);
     mkdirSync(stateDir, {
         recursive: true
     });
     const dbPath = join(stateDir, dialect.filename);
-    const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);
+    const { handle, conn } = await connectUnlocked(dbPath, cfg, configDir, dialect);
     // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on
     // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.
     let closed = false;
     // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool
     // with the build() call that follows them.
-    const builder = createBuilder(conn, cfg, (_cfg_rootDir = cfg.rootDir) !== null && _cfg_rootDir !== void 0 ? _cfg_rootDir : cfg.baseDir, dialect.reconcileDialect);
+    const builder = createBuilder(conn, cfg, {
+        rootDir: (_cfg_rootDir = cfg.rootDir) !== null && _cfg_rootDir !== void 0 ? _cfg_rootDir : cfg.baseDir,
+        configDir
+    }, dialect.reconcileDialect);
     try {
         await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
         // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an