sensemaking 0.27.0 → 0.28.0

package/dist/esm/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates\n// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);\n// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over\n// 300 columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,kGAAkG;AAClG,iGAAiG;AACjG,kGAAkG;AAClG,2BAA2B;AAC3B,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates\n// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);\n// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over\n// 300 columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,kGAAkG;AAClG,iGAAiG;AACjG,kGAAkG;AAClG,2BAA2B;AAC3B,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}

package/dist/esm/store/duckdb/lexical.d.ts

@@ -1,14 +1,9 @@
 import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
-interface FtsIndexState {
-    stale: boolean;
-    orderedMacroReady: boolean;
-}
 export declare function markContentStale(conn: Connection): Promise<void>;
 export declare function prepareFts(conn: Connection): Promise<void>;
 export declare function assertFtsReady(conn: Connection): Promise<void>;
-export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
+export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;
     markStale: () => Promise<void>;
 };
-export {};

package/dist/esm/store/duckdb/lexical.js

@@ -4,14 +4,15 @@ import { SenseError } from '../../errors.js';
 import { foldForSearch, matchesSearchPhrase, searchTokens } from '../../text/segment.js';
 import { getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
-import { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.js';
+import { duckdbFts } from './engine.js';
+import { nativeQueryTerms, orderedBm25Sql } from './ordered-bm25.js';
 const FIELDS = [
     'title',
     'summary',
     'text'
 ];
 // Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no
-// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.
+// field_weights argument (verified 1.4.5 through 1.5.6), so each field is scored separately and combined here.
 const FIELD_WEIGHT = {
     title: 10,
     summary: 5,
@@ -68,8 +69,7 @@ async function stateFor(conn) {
         // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that
         // never built an index, or one a prior process marked stale and never got to clear.
         state = {
-            stale: await getMeta(conn, FTS_STALE_META_KEY) !== '0',
-            orderedMacroReady: false
+            stale: await getMeta(conn, FTS_STALE_META_KEY) !== '0'
         };
         ftsState.set(conn, state);
     }
@@ -90,8 +90,7 @@ async function hasFtsArtifact(conn) {
 // transaction (reconcile.ts) so a crash never lands the content write without the stale mark.
 export async function markContentStale(conn) {
     ftsState.set(conn, {
-        stale: true,
-        orderedMacroReady: false
+        stale: true
     });
     await setMeta(conn, FTS_STALE_META_KEY, '1');
 }
@@ -146,11 +145,8 @@ async function rebuildFts(conn, state) {
     await conn.exec('INSTALL fts; LOAD fts;');
     if (!state.stale && await hasFtsArtifact(conn)) return;
     state.stale = true;
-    state.orderedMacroReady = false;
     await withTransaction(conn, async ()=>{
-        // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's
-        // default 571-word list would otherwise drop common words (e.g. "and") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.
-        await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);
+        await conn.exec((await duckdbFts()).createIndex());
         // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant
         // rebuild next time), never a cleared mark serving a half-built index as fresh.
         await setMeta(conn, FTS_STALE_META_KEY, '0');
@@ -175,24 +171,23 @@ export async function assertFtsReady(conn) {
 }
 // Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
 // score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
-function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
+function buildScoreAndGate(fts, words, nativeTerms, bareWords, phraseWords, substrings) {
     const scoreParts = [];
     const gateParts = [];
     const scoreParams = [];
     const gateParams = [];
     if (words.length > 0) {
-        const wordQuery = words.map(foldForSearch).join(' ');
         for (const field of FIELDS){
-            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
-            scoreParams.push(wordQuery);
+            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${orderedBm25Sql(fts, field, nativeTerms.length)}, 0)`);
+            scoreParams.push(...nativeTerms);
         }
         if (bareWords.length > 0) {
-            const nativeGate = `${ORDERED_BM25_MACRO}(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const nativeGate = fts.conjunctiveGate();
             gateParts.push(nativeGate);
             gateParams.push(bareWords.map(foldForSearch).join(' '));
         }
         if (phraseWords.length > 0) {
-            const nativeGate = `${ORDERED_BM25_MACRO}(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const nativeGate = fts.conjunctiveGate();
             const fallback = phraseWords.map((_word)=>`(${FIELDS.map((field)=>`contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');
             gateParts.push(`(${nativeGate} OR (${fallback}))`);
             gateParams.push(phraseWords.map(foldForSearch).join(' '));
@@ -219,7 +214,7 @@ function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
 }
 // Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).
 // Phrase rows carry authored fields through the post-filter; the public result still contains paths only.
-export async function queryLexical(conn, terms, opts, state) {
+export async function queryLexical(conn, terms, opts) {
     const unsupported = unsupportedOperator(terms);
     if (unsupported !== null) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "duckdb" does not implement FTS5's ${unsupported.label} ("${unsupported.token}") in this build; rephrase "${terms.trim()}" without it, or set "store" to "sqlite" in this tree's config to search it as written`);
@@ -228,8 +223,9 @@ export async function queryLexical(conn, terms, opts, state) {
     const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);
     if (emptyPhrase || words.length === 0 && substrings.length === 0) return [];
     if (words.length > 0) await assertFtsReady(conn);
-    if (words.length > 0) await ensureOrderedBm25(conn, state);
-    const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);
+    const fts = await duckdbFts();
+    const nativeTerms = words.length > 0 ? await nativeQueryTerms(conn, fts, words.map(foldForSearch).join(' ')) : [];
+    const { scoreSql, gateSql, params } = buildScoreAndGate(fts, words, nativeTerms, bareWords, phraseWords, substrings);
     const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
     const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';
     const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (
@@ -255,7 +251,7 @@ export async function queryLexical(conn, terms, opts, state) {
 // otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.
 export function createLexicalIndex(conn) {
     return {
-        query: async (terms, opts)=>queryLexical(conn, terms, opts, await stateFor(conn)),
+        query: async (terms, opts)=>queryLexical(conn, terms, opts),
         markStale: ()=>markContentStale(conn)
     };
 }

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n  orderedMacroReady: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0', orderedMacroReady: false };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\nasync function hasFtsArtifact(conn: Connection): Promise<boolean> {\n  const stmt = await conn.prepare(`\n    SELECT 1 AS ready\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content'\n      AND function_name = 'match_bm25'\n      AND function_type = 'macro'\n    LIMIT 1\n  `);\n  return (await stmt.get()) !== undefined;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true, orderedMacroReady: false });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function rebuildFts(conn: Connection, state: FtsIndexState): Promise<void> {\n  await conn.exec('INSTALL fts; LOAD fts;');\n  if (!state.stale && (await hasFtsArtifact(conn))) return;\n  state.stale = true;\n  state.orderedMacroReady = false;\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\nexport async function prepareFts(conn: Connection): Promise<void> {\n  await rebuildFts(conn, await stateFor(conn));\n}\n\nexport async function assertFtsReady(conn: Connection): Promise<void> {\n  const state = await stateFor(conn);\n  if (state.stale) throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index is missing or stale; run `sense build` (or library build(config)) before lexical querying with build disabled');\n  try {\n    await conn.exec('LOAD fts;');\n    if (!(await hasFtsArtifact(conn))) {\n      throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index artifact is missing; run `sense build` (or library build(config)) before lexical querying with build disabled');\n    }\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('INDEX_NOT_READY', `the DuckDB lexical extension is not prepared (${(err as Error).message}); run \\`sense build\\` (or library build(config))`);\n  }\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await assertFtsReady(conn);\n  if (words.length > 0) await ensureOrderedBm25(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","ensureOrderedBm25","ORDERED_BM25_MACRO","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","orderedMacroReady","set","hasFtsArtifact","stmt","prepare","undefined","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","rebuildFts","exec","prepareFts","assertFtsReady","err","message","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAE1E,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAOA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM5B,QAAQyB,MAAMJ,wBAAyB;YAAKQ,mBAAmB;QAAM;QAC7FP,SAASQ,GAAG,CAACL,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,eAAeK,eAAeN,IAAgB;IAC5C,MAAMO,OAAO,MAAMP,KAAKQ,OAAO,CAAC,CAAC;;;;;;;EAOjC,CAAC;IACD,OAAO,AAAC,MAAMD,KAAKL,GAAG,OAAQO;AAChC;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeC,iBAAiBV,IAAgB;IACrDH,SAASQ,GAAG,CAACL,MAAM;QAAEG,OAAO;QAAMC,mBAAmB;IAAM;IAC3D,MAAM5B,QAAQwB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASe,WAAWtB,KAAa;IAC/B,MAAMuB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAM3B,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAAC2B,IAAIC;QACtD,MAAMC,SAASD,MAAMxB,IAAI;QACzB,MAAM0B,eAAe/C,aAAa8C;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAM3B,SAAS2B,aAAc;YAChC,IAAI3B,MAAM8B,QAAQ,EAAET,WAAWQ,IAAI,CAAC7B,MAAMV,IAAI;iBACzC;gBACH4B,MAAMW,IAAI,CAAC7B,MAAMV,IAAI;gBACrB8B,YAAYS,IAAI,CAAC7B,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMyC,OAAOnC,eAAeoC,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAM5B,SAASpB,aAAamD,KAAM;YACrC,IAAI/B,MAAM8B,QAAQ,EAAET,WAAWQ,IAAI,CAAC7B,MAAMV,IAAI;iBACzC;gBACH4B,MAAMW,IAAI,CAAC7B,MAAMV,IAAI;gBACrB6B,UAAUU,IAAI,CAAC7B,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAE4B;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,WAAW3B,IAAgB,EAAEC,KAAoB;IAC9D,MAAMD,KAAK4B,IAAI,CAAC;IAChB,IAAI,CAAC3B,MAAME,KAAK,IAAK,MAAMG,eAAeN,OAAQ;IAClDC,MAAME,KAAK,GAAG;IACdF,MAAMG,iBAAiB,GAAG;IAC1B,MAAM3B,gBAAgBuB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAK4B,IAAI,CAAC,CAAC,qGAAqG,CAAC;QACvH,0FAA0F;QAC1F,gFAAgF;QAChF,MAAMpD,QAAQwB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,OAAO,eAAe0B,WAAW7B,IAAgB;IAC/C,MAAM2B,WAAW3B,MAAM,MAAMD,SAASC;AACxC;AAEA,OAAO,eAAe8B,eAAe9B,IAAgB;IACnD,MAAMC,QAAQ,MAAMF,SAASC;IAC7B,IAAIC,MAAME,KAAK,EAAE,MAAM,IAAIhC,WAAW,mBAAmB;IACzD,IAAI;QACF,MAAM6B,KAAK4B,IAAI,CAAC;QAChB,IAAI,CAAE,MAAMtB,eAAeN,OAAQ;YACjC,MAAM,IAAI7B,WAAW,mBAAmB;QAC1C;IACF,EAAE,OAAO4D,KAAK;QACZ,IAAIA,eAAe5D,YAAY,MAAM4D;QACrC,MAAM,IAAI5D,WAAW,mBAAmB,CAAC,8CAA8C,EAAE,AAAC4D,IAAcC,OAAO,CAAC,iDAAiD,CAAC;IACpK;AACF;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASC,kBAAkBrB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,MAAMmB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIzB,MAAMU,MAAM,GAAG,GAAG;QACpB,MAAMgB,YAAY1B,MAAM2B,GAAG,CAACnE,eAAeoE,IAAI,CAAC;QAChD,KAAK,MAAMC,SAAS7D,OAAQ;YAC1BsD,WAAWX,IAAI,CAAC,GAAG1C,YAAY,CAAC4D,MAAM,CAAC,cAAc,EAAE9D,mBAAmB,+BAA+B,EAAE8D,MAAM,4BAA4B,CAAC;YAC9IL,YAAYb,IAAI,CAACe;QACnB;QACA,IAAIzB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMoB,aAAa,GAAG/D,mBAAmB,oDAAoD,CAAC;YAC9FwD,UAAUZ,IAAI,CAACmB;YACfL,WAAWd,IAAI,CAACV,UAAU0B,GAAG,CAACnE,eAAeoE,IAAI,CAAC;QACpD;QACA,IAAI1B,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMoB,aAAa,GAAG/D,mBAAmB,oDAAoD,CAAC;YAC9F,MAAMgE,WAAW7B,YAAYyB,GAAG,CAAC,CAACK,QAAU,CAAC,CAAC,EAAEhE,OAAO2D,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJL,UAAUZ,IAAI,CAAC,CAAC,CAAC,EAAEmB,WAAW,KAAK,EAAEC,SAAS,EAAE,CAAC;YACjDN,WAAWd,IAAI,CAACT,YAAYyB,GAAG,CAACnE,eAAeoE,IAAI,CAAC;YACpDH,WAAWd,IAAI,IAAIT,YAAY+B,OAAO,CAAC,CAACC,OAASlE,OAAO2D,GAAG,CAAC,IAAMnE,cAAc0E;QAClF;IACF;IAEA,KAAK,MAAMC,OAAOhC,WAAY;QAC5B,MAAMiC,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMR,SAAS7D,OAAQ;YAC1BsD,WAAWX,IAAI,CAAC,GAAG1C,YAAY,CAAC4D,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHL,YAAYb,IAAI,CAACnD,cAAc4E;QACjC;QACAb,UAAUZ,IAAI,CAAC,CAAC,CAAC,EAAE3C,OAAO2D,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GH,WAAWd,IAAI,IAAI3C,OAAO2D,GAAG,CAAC,IAAMnE,cAAc4E;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWM,IAAI,CAAC;QAAQW,SAAShB,UAAUK,IAAI,CAAC,YAAY;QAAQY,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAarD,IAAgB,EAAEX,KAAa,EAAEiE,IAAyB,EAAErD,KAAoB;IACjH,MAAMsD,cAAcnE,oBAAoBC;IACxC,IAAIkE,gBAAgB,MAAM;QACxB,MAAM,IAAIpF,WAAW,4BAA4B,CAAC,yCAAyC,EAAEoF,YAAYrE,KAAK,CAAC,GAAG,EAAEqE,YAAY7D,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAE6D,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE1C,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWtB;IACvF,IAAI4B,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMQ,eAAe9B;IAC3C,IAAIY,MAAMU,MAAM,GAAG,GAAG,MAAM5C,kBAAkBsB,MAAMC;IAEpD,MAAM,EAAEiD,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGnB,kBAAkBrB,OAAOC,WAAWC,aAAaC;IACvF,MAAM6C,eAAe5C,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMuC,WAAW7C,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAMwC,MAAM,CAAC,kBAAkB,EAAE9C,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAE4B,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAMtD,OAAO,MAAMP,KAAKQ,OAAO,CAACsD;IAChC,MAAMC,OAAQ,MAAMxD,KAAKyD,GAAG,IAAIZ,WAAYpC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACqC;KAAM;IAC7E,MAAMM,UAAUjD,QAAQM,MAAM,KAAK,IAAIyC,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQnD,QAAQoD,KAAK,CAAC,CAAChD,SAAWxC,OAAOyF,IAAI,CAAC,CAAC5B;oBAA8B0B;uBAApB9F,qBAAoB8F,aAAAA,GAAG,CAAC1B,MAAM,cAAV0B,wBAAAA,aAAc,IAAI/C;;IAC1J,OAAO6C,QAAQK,KAAK,CAAC,GAAGX,OAAOpB,GAAG,CAAC,CAAC,EAAEgC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmBxE,IAAgB;IACjD,OAAO;QACLyE,OAAO,OAAOpF,OAAOiE,OAASD,aAAarD,MAAMX,OAAOiE,MAAM,MAAMvD,SAASC;QAC7E0E,WAAW,IAAMhE,iBAAiBV;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { type DuckdbFtsBuilders, duckdbFts } from './engine.ts';\nimport { nativeQueryTerms, orderedBm25Sql } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.4.5 through 1.5.6), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0' };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\nasync function hasFtsArtifact(conn: Connection): Promise<boolean> {\n  const stmt = await conn.prepare(`\n    SELECT 1 AS ready\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content'\n      AND function_name = 'match_bm25'\n      AND function_type = 'macro'\n    LIMIT 1\n  `);\n  return (await stmt.get()) !== undefined;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function rebuildFts(conn: Connection, state: FtsIndexState): Promise<void> {\n  await conn.exec('INSTALL fts; LOAD fts;');\n  if (!state.stale && (await hasFtsArtifact(conn))) return;\n  state.stale = true;\n  await withTransaction(conn, async () => {\n    await conn.exec((await duckdbFts()).createIndex());\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\nexport async function prepareFts(conn: Connection): Promise<void> {\n  await rebuildFts(conn, await stateFor(conn));\n}\n\nexport async function assertFtsReady(conn: Connection): Promise<void> {\n  const state = await stateFor(conn);\n  if (state.stale) throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index is missing or stale; run `sense build` (or library build(config)) before lexical querying with build disabled');\n  try {\n    await conn.exec('LOAD fts;');\n    if (!(await hasFtsArtifact(conn))) {\n      throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index artifact is missing; run `sense build` (or library build(config)) before lexical querying with build disabled');\n    }\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('INDEX_NOT_READY', `the DuckDB lexical extension is not prepared (${(err as Error).message}); run \\`sense build\\` (or library build(config))`);\n  }\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(fts: DuckdbFtsBuilders, words: string[], nativeTerms: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${orderedBm25Sql(fts, field, nativeTerms.length)}, 0)`);\n      scoreParams.push(...nativeTerms);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = fts.conjunctiveGate();\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = fts.conjunctiveGate();\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await assertFtsReady(conn);\n  const fts = await duckdbFts();\n  const nativeTerms = words.length > 0 ? await nativeQueryTerms(conn, fts, words.map(foldForSearch).join(' ')) : [];\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(fts, words, nativeTerms, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","duckdbFts","nativeQueryTerms","orderedBm25Sql","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","set","hasFtsArtifact","stmt","prepare","undefined","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","rebuildFts","exec","createIndex","prepareFts","assertFtsReady","err","message","buildScoreAndGate","fts","nativeTerms","scoreParts","gateParts","scoreParams","gateParams","field","nativeGate","conjunctiveGate","map","join","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAAiCC,SAAS,QAAQ,cAAc;AAChE,SAASC,gBAAgB,EAAEC,cAAc,QAAQ,oBAAoB;AAErE,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,+GAA+G;AAC/G,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM7B,QAAQ0B,MAAMJ,wBAAyB;QAAI;QACnEC,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,eAAeI,eAAeL,IAAgB;IAC5C,MAAMM,OAAO,MAAMN,KAAKO,OAAO,CAAC,CAAC;;;;;;;EAOjC,CAAC;IACD,OAAO,AAAC,MAAMD,KAAKJ,GAAG,OAAQM;AAChC;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeC,iBAAiBT,IAAgB;IACrDH,SAASO,GAAG,CAACJ,MAAM;QAAEG,OAAO;IAAK;IACjC,MAAM5B,QAAQyB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASc,WAAWrB,KAAa;IAC/B,MAAMsB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAM1B,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAAC0B,IAAIC;QACtD,MAAMC,SAASD,MAAMvB,IAAI;QACzB,MAAMyB,eAAe/C,aAAa8C;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAM1B,SAAS0B,aAAc;YAChC,IAAI1B,MAAM6B,QAAQ,EAAET,WAAWQ,IAAI,CAAC5B,MAAMV,IAAI;iBACzC;gBACH2B,MAAMW,IAAI,CAAC5B,MAAMV,IAAI;gBACrB6B,YAAYS,IAAI,CAAC5B,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMwC,OAAOlC,eAAemC,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAM3B,SAASrB,aAAamD,KAAM;YACrC,IAAI9B,MAAM6B,QAAQ,EAAET,WAAWQ,IAAI,CAAC5B,MAAMV,IAAI;iBACzC;gBACH2B,MAAMW,IAAI,CAAC5B,MAAMV,IAAI;gBACrB4B,UAAUU,IAAI,CAAC5B,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAE2B;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,WAAW1B,IAAgB,EAAEC,KAAoB;IAC9D,MAAMD,KAAK2B,IAAI,CAAC;IAChB,IAAI,CAAC1B,MAAME,KAAK,IAAK,MAAME,eAAeL,OAAQ;IAClDC,MAAME,KAAK,GAAG;IACd,MAAM3B,gBAAgBwB,MAAM;QAC1B,MAAMA,KAAK2B,IAAI,CAAC,AAAC,CAAA,MAAMlD,WAAU,EAAGmD,WAAW;QAC/C,0FAA0F;QAC1F,gFAAgF;QAChF,MAAMrD,QAAQyB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,OAAO,eAAe0B,WAAW7B,IAAgB;IAC/C,MAAM0B,WAAW1B,MAAM,MAAMD,SAASC;AACxC;AAEA,OAAO,eAAe8B,eAAe9B,IAAgB;IACnD,MAAMC,QAAQ,MAAMF,SAASC;IAC7B,IAAIC,MAAME,KAAK,EAAE,MAAM,IAAIjC,WAAW,mBAAmB;IACzD,IAAI;QACF,MAAM8B,KAAK2B,IAAI,CAAC;QAChB,IAAI,CAAE,MAAMtB,eAAeL,OAAQ;YACjC,MAAM,IAAI9B,WAAW,mBAAmB;QAC1C;IACF,EAAE,OAAO6D,KAAK;QACZ,IAAIA,eAAe7D,YAAY,MAAM6D;QACrC,MAAM,IAAI7D,WAAW,mBAAmB,CAAC,8CAA8C,EAAE,AAAC6D,IAAcC,OAAO,CAAC,iDAAiD,CAAC;IACpK;AACF;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASC,kBAAkBC,GAAsB,EAAEvB,KAAe,EAAEwB,WAAqB,EAAEvB,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IACzJ,MAAMsB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAI5B,MAAMU,MAAM,GAAG,GAAG;QACpB,KAAK,MAAMmB,SAAS5D,OAAQ;YAC1BwD,WAAWd,IAAI,CAAC,GAAGzC,YAAY,CAAC2D,MAAM,CAAC,cAAc,EAAE7D,eAAeuD,KAAKM,OAAOL,YAAYd,MAAM,EAAE,IAAI,CAAC;YAC3GiB,YAAYhB,IAAI,IAAIa;QACtB;QACA,IAAIvB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMoB,aAAaP,IAAIQ,eAAe;YACtCL,UAAUf,IAAI,CAACmB;YACfF,WAAWjB,IAAI,CAACV,UAAU+B,GAAG,CAACxE,eAAeyE,IAAI,CAAC;QACpD;QACA,IAAI/B,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMoB,aAAaP,IAAIQ,eAAe;YACtC,MAAMG,WAAWhC,YAAY8B,GAAG,CAAC,CAACG,QAAU,CAAC,CAAC,EAAElE,OAAO+D,GAAG,CAAC,CAACH,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAEI,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJP,UAAUf,IAAI,CAAC,CAAC,CAAC,EAAEmB,WAAW,KAAK,EAAEI,SAAS,EAAE,CAAC;YACjDN,WAAWjB,IAAI,CAACT,YAAY8B,GAAG,CAACxE,eAAeyE,IAAI,CAAC;YACpDL,WAAWjB,IAAI,IAAIT,YAAYkC,OAAO,CAAC,CAACC,OAASpE,OAAO+D,GAAG,CAAC,IAAMxE,cAAc6E;QAClF;IACF;IAEA,KAAK,MAAMC,OAAOnC,WAAY;QAC5B,MAAMoC,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMX,SAAS5D,OAAQ;YAC1BwD,WAAWd,IAAI,CAAC,GAAGzC,YAAY,CAAC2D,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHF,YAAYhB,IAAI,CAACnD,cAAc+E;QACjC;QACAb,UAAUf,IAAI,CAAC,CAAC,CAAC,EAAE1C,OAAO+D,GAAG,CAAC,CAACH,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAEI,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GL,WAAWjB,IAAI,IAAI1C,OAAO+D,GAAG,CAAC,IAAMxE,cAAc+E;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWQ,IAAI,CAAC;QAAQS,SAAShB,UAAUO,IAAI,CAAC,YAAY;QAAQU,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAavD,IAAgB,EAAEX,KAAa,EAAEmE,IAAyB;IAC3F,MAAMC,cAAcrE,oBAAoBC;IACxC,IAAIoE,gBAAgB,MAAM;QACxB,MAAM,IAAIvF,WAAW,4BAA4B,CAAC,yCAAyC,EAAEuF,YAAYvE,KAAK,CAAC,GAAG,EAAEuE,YAAY/D,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAE+D,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE7C,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWrB;IACvF,IAAI2B,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMS,eAAe9B;IAC3C,MAAMkC,MAAM,MAAMzD;IAClB,MAAM0D,cAAcxB,MAAMU,MAAM,GAAG,IAAI,MAAM3C,iBAAiBsB,MAAMkC,KAAKvB,MAAMgC,GAAG,CAACxE,eAAeyE,IAAI,CAAC,QAAQ,EAAE;IAEjH,MAAM,EAAEQ,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGrB,kBAAkBC,KAAKvB,OAAOwB,aAAavB,WAAWC,aAAaC;IACzG,MAAMgD,eAAe/C,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAM0C,WAAWhD,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAM2C,MAAM,CAAC,kBAAkB,EAAEjD,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAE+B,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAMzD,OAAO,MAAMN,KAAKO,OAAO,CAACyD;IAChC,MAAMC,OAAQ,MAAM3D,KAAK4D,GAAG,IAAIZ,WAAYvC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACwC;KAAM;IAC7E,MAAMM,UAAUpD,QAAQM,MAAM,KAAK,IAAI4C,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQtD,QAAQuD,KAAK,CAAC,CAACnD,SAAWvC,OAAO2F,IAAI,CAAC,CAAC/B;oBAA8B6B;uBAApBjG,qBAAoBiG,aAAAA,GAAG,CAAC7B,MAAM,cAAV6B,wBAAAA,aAAc,IAAIlD;;IAC1J,OAAOgD,QAAQK,KAAK,CAAC,GAAGX,OAAOlB,GAAG,CAAC,CAAC,EAAE8B,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmB1E,IAAgB;IACjD,OAAO;QACL2E,OAAO,OAAOtF,OAAOmE,OAASD,aAAavD,MAAMX,OAAOmE;QACxDoB,WAAW,IAAMnE,iBAAiBT;IACpC;AACF"}

package/dist/esm/store/duckdb/native.d.ts

@@ -1,4 +1,6 @@
+import { type NativeModule } from '../native.js';
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
-export declare const DUCKDB_VERSION = "1.5.5-r.5";
-export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.5";
-export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;
+type DuckdbApi = typeof import('@duckdb/node-api');
+export declare function duckdbNative(): Promise<NativeModule<DuckdbApi>>;
+export declare function duckdbApi(): Promise<DuckdbApi>;
+export {};

package/dist/esm/store/duckdb/native.js

@@ -1,18 +1,20 @@
-import { loadOrInstall, packageNodeModules } from '../native.js';
+import { loadOrInstall, packageNodeModules, peerRange } from '../native.js';
 export const DUCKDB_PACKAGE = '@duckdb/node-api';
-export const DUCKDB_VERSION = '1.5.5-r.5';
-export const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;
-const DUCKDB = {
-    store: 'duckdb',
-    pkg: DUCKDB_PACKAGE,
-    sizeHint: '~110MB',
-    version: DUCKDB_VERSION,
-    installSpec: DUCKDB_INSTALL_SPEC
-};
-let duckdbApiPromise;
+let duckdbNativePromise;
 // The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning
 // the pre-install "not found" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.
-export function duckdbApi() {
-    if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall(DUCKDB, packageNodeModules());
-    return duckdbApiPromise;
+export function duckdbNative() {
+    if (!duckdbNativePromise) {
+        const descriptor = {
+            store: 'duckdb',
+            pkg: DUCKDB_PACKAGE,
+            sizeHint: '~110MB',
+            range: peerRange(DUCKDB_PACKAGE)
+        };
+        duckdbNativePromise = loadOrInstall(descriptor, packageNodeModules());
+    }
+    return duckdbNativePromise;
+}
+export async function duckdbApi() {
+    return (await duckdbNative()).api;
 }

package/dist/esm/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_VERSION = '1.5.5-r.5';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  version: DUCKDB_VERSION,\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB_VERSION","DUCKDB_INSTALL_SPEC","DUCKDB","store","pkg","sizeHint","version","installSpec","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AACjD,OAAO,MAAMC,iBAAiB,YAAY;AAC1C,OAAO,MAAMC,sBAAsB,GAAGF,eAAe,CAAC,EAAEC,gBAAgB,CAAC;AAEzE,MAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKL;IACLM,UAAU;IACVC,SAASN;IACTO,aAAaN;AACf;AAEA,IAAIO;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBX,cAAiDK,QAAQJ;IACnG,OAAOU;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, type NativeModule, packageNodeModules, peerRange } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\ntype DuckdbApi = typeof import('@duckdb/node-api');\n\nlet duckdbNativePromise: Promise<NativeModule<DuckdbApi>> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbNative(): Promise<NativeModule<DuckdbApi>> {\n  if (!duckdbNativePromise) {\n    const descriptor: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB', range: peerRange(DUCKDB_PACKAGE) };\n    duckdbNativePromise = loadOrInstall<DuckdbApi>(descriptor, packageNodeModules());\n  }\n  return duckdbNativePromise;\n}\n\nexport async function duckdbApi(): Promise<DuckdbApi> {\n  return (await duckdbNative()).api;\n}\n"],"names":["loadOrInstall","packageNodeModules","peerRange","DUCKDB_PACKAGE","duckdbNativePromise","duckdbNative","descriptor","store","pkg","sizeHint","range","duckdbApi","api"],"mappings":"AAAA,SAASA,aAAa,EAA4CC,kBAAkB,EAAEC,SAAS,QAAQ,eAAe;AAEtH,OAAO,MAAMC,iBAAiB,mBAAmB;AAIjD,IAAIC;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,qBAAqB;QACxB,MAAME,aAA+B;YAAEC,OAAO;YAAUC,KAAKL;YAAgBM,UAAU;YAAUC,OAAOR,UAAUC;QAAgB;QAClIC,sBAAsBJ,cAAyBM,YAAYL;IAC7D;IACA,OAAOG;AACT;AAEA,OAAO,eAAeO;IACpB,OAAO,AAAC,CAAA,MAAMN,cAAa,EAAGO,GAAG;AACnC"}

package/dist/esm/store/duckdb/open.js

@@ -1,11 +1,13 @@
 import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures } from '../../features/index.js';
+import { ENGINE_CHECK_DOCS, engineIdentity, npmRemedy, checkEngine as runEngineCheck } from '../engine.js';
+import { peerRange } from '../native.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
-import { assertFtsReady, prepareFts } from './lexical.js';
-import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
+import { assertFtsReady, prepareFts, queryLexical } from './lexical.js';
+import { DUCKDB_PACKAGE, duckdbApi, duckdbNative } from './native.js';
 import { duckdbDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
@@ -89,6 +91,40 @@ async function connect(dbPath, _cfg, options) {
         throw err;
     }
 }
+async function duckdbEngineIdentity() {
+    return engineIdentity(DUCKDB_PACKAGE, (await duckdbNative()).version);
+}
+// The real schema, index build and lexical query over store/engine.ts's fixture, in memory.
+async function checkDuckdbEngine() {
+    const { api, version } = await duckdbNative();
+    const instance = await api.DuckDBInstance.create(':memory:');
+    const handle = {
+        instance,
+        duckdb: await instance.connect()
+    };
+    try {
+        const conn = createConnection(handle.duckdb);
+        await conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)');
+        await ensureCoreSchema(handle, conn);
+        await conn.runBatch('INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)', ENGINE_CHECK_DOCS.map((doc)=>[
+                doc.path,
+                doc.title,
+                doc.summary,
+                doc.text
+            ]));
+        await runEngineCheck('duckdb', engineIdentity(DUCKDB_PACKAGE, version), npmRemedy(DUCKDB_PACKAGE, peerRange(DUCKDB_PACKAGE)), async (terms)=>{
+            await prepareFts(conn);
+            return (await queryLexical(conn, terms, {
+                whereJoin: '',
+                whereCond: '',
+                scopeCond: '',
+                limit: 10
+            })).map((hit)=>hit.path);
+        });
+    } finally{
+        await close(handle);
+    }
+}
 // This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.
 export const duckdbOpenDialect = {
     filename: DB_FILENAME,
@@ -100,6 +136,8 @@ export const duckdbOpenDialect = {
     // way: posix "Could not set lock on file ... Conflicting lock is held in <exe> (PID n)", Windows
     // "Cannot open file ... being used by another process". Both are the same condition.
     isLocked: (err)=>/Could not set lock on file|being used by another process/.test(err.message),
+    engineIdentity: duckdbEngineIdentity,
+    checkEngine: checkDuckdbEngine,
     ensureCoreSchema,
     ensureFeatureSchema,
     prepareConnection,

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '6';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureCoreSchema(_handle: DuckdbHandle, conn: Connection): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\nasync function ensureFeatureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n}\n\nasync function prepareConnection(handle: DuckdbHandle, _conn: Connection, _cfg: Config): Promise<void> {\n  await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureCoreSchema,\n  ensureFeatureSchema,\n  prepareConnection,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["STORE_DIMS","SenseError","activeFeatures","openWithDialect","getMeta","setMeta","createConnection","assertFtsReady","prepareFts","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureCoreSchema","_handle","conn","exec","ensureFeatureSchema","cfg","feature","name","schema","prepareConnection","handle","_conn","_cfg","duckdb","close","disconnectSync","instance","closeSync","connect","dbPath","options","DuckDBInstance","err","message","create","observational","access_mode","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","prepareLexical","assertLexicalReady","openDuckdb"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,UAAU,QAAQ,eAAe;AAC1D,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,eAAeC,iBAAiBC,OAAqB,EAAEC,IAAgB;IACrE,MAAMA,KAAKC,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMD,KAAKC,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMD,KAAKC,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,MAAMD,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,IAAI,AAAC,MAAMhB,QAAQe,MAAM,sBAAuB,MAAM,MAAMd,QAAQc,MAAM,kBAAkBH;AAC9F;AAEA,eAAeK,oBAAoBH,OAAqB,EAAEC,IAAgB,EAAEG,GAAW;IACrF,KAAK,MAAMC,WAAWrB,eAAeoB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIC,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKC,IAAI,CAAC,CAAC,kKAAkK,EAAEpB,WAAW,+BAA+B,CAAC;YAChO;QACF;QACA,MAAMuB,QAAQE,MAAM,CAACN;IACvB;AACF;AAEA,eAAeO,kBAAkBC,MAAoB,EAAEC,KAAiB,EAAEC,IAAY;IACpF,MAAMf,uBAAuBa,OAAOG,MAAM,EAAE9B;AAC9C;AAEA,iFAAiF;AACjF,eAAe+B,MAAMJ,MAAoB;IACvCA,OAAOG,MAAM,CAACE,cAAc;IAC5BL,OAAOM,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEP,IAAoB,EAAEQ,OAA0D;IACrH,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAM5B,WAAU;IACxC,EAAE,OAAO6B,KAAK;QACZ,IAAIA,eAAetC,YAAY,MAAMsC;QACrC,MAAM,IAAItC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEQ,eAAe,UAAU,EAAE,AAAC8B,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIV;IACJ,IAAIG;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQC,QAAQK,aAAa,GAAG;YAAEC,aAAa;QAAY,IAAI;YAAEC,+BAA+B;QAAS;QAChJd,SAAS,MAAMG,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEmC,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAM5B,kBAAkBkB;QACxB,MAAMX,OAAOb,iBAAiBwB;QAC9B,OAAO;YAAEH,QAAQ;gBAAEM;gBAAUH;YAAO;YAAGX;QAAK;IAC9C,EAAE,OAAOoB,KAAK;QACZ,MAAMR,MAAM;YAAEE;YAAUH;QAAO;QAC/B,MAAMS;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMM,oBAA+C;IAC1DC,UAAU/B;IACVgC,eAAe/B;IACfgC,kBAAkBrC;IAClBwB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACV,MAAQ,2DAA2DW,IAAI,CAACX,IAAIC,OAAO;IAC9FvB;IACAI;IACAK;IACAyB,gBAAgB3C;IAChB4C,oBAAoB7C;IACpBM,aAAa,CAACc,QAAQR,OAASN,YAAYc,OAAOM,QAAQ,EAAEN,OAAOG,MAAM,EAAEX;AAC7E,EAAE;AAEF,OAAO,eAAekC,WAAW/B,GAAmB,EAAEe,OAA6B;IACjF,OAAOlC,gBAAgBmB,KAAKuB,mBAAmBR;AACjD"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport { ENGINE_CHECK_DOCS, engineIdentity, npmRemedy, checkEngine as runEngineCheck } from '../engine.ts';\nimport { peerRange } from '../native.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts, queryLexical } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi, duckdbNative } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '6';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureCoreSchema(_handle: DuckdbHandle, conn: Connection): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\nasync function ensureFeatureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n}\n\nasync function prepareConnection(handle: DuckdbHandle, _conn: Connection, _cfg: Config): Promise<void> {\n  await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\nasync function duckdbEngineIdentity(): Promise<string> {\n  return engineIdentity(DUCKDB_PACKAGE, (await duckdbNative()).version);\n}\n\n// The real schema, index build and lexical query over store/engine.ts's fixture, in memory.\nasync function checkDuckdbEngine(): Promise<void> {\n  const { api, version } = await duckdbNative();\n  const instance = await api.DuckDBInstance.create(':memory:');\n  const handle = { instance, duckdb: await instance.connect() };\n  try {\n    const conn = createConnection(handle.duckdb);\n    await conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)');\n    await ensureCoreSchema(handle, conn);\n    await conn.runBatch(\n      'INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)',\n      ENGINE_CHECK_DOCS.map((doc) => [doc.path, doc.title, doc.summary, doc.text])\n    );\n    await runEngineCheck('duckdb', engineIdentity(DUCKDB_PACKAGE, version), npmRemedy(DUCKDB_PACKAGE, peerRange(DUCKDB_PACKAGE)), async (terms) => {\n      await prepareFts(conn);\n      return (await queryLexical(conn, terms, { whereJoin: '', whereCond: '', scopeCond: '', limit: 10 })).map((hit) => hit.path);\n    });\n  } finally {\n    await close(handle);\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  engineIdentity: duckdbEngineIdentity,\n  checkEngine: checkDuckdbEngine,\n  ensureCoreSchema,\n  ensureFeatureSchema,\n  prepareConnection,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["STORE_DIMS","SenseError","activeFeatures","ENGINE_CHECK_DOCS","engineIdentity","npmRemedy","checkEngine","runEngineCheck","peerRange","openWithDialect","getMeta","setMeta","createConnection","assertFtsReady","prepareFts","queryLexical","DUCKDB_PACKAGE","duckdbApi","duckdbNative","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureCoreSchema","_handle","conn","exec","ensureFeatureSchema","cfg","feature","name","schema","prepareConnection","handle","_conn","_cfg","duckdb","close","disconnectSync","instance","closeSync","connect","dbPath","options","DuckDBInstance","err","message","create","observational","access_mode","storage_compatibility_version","duckdbEngineIdentity","version","checkDuckdbEngine","api","runBatch","map","doc","path","title","summary","text","terms","whereJoin","whereCond","scopeCond","limit","hit","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","prepareLexical","assertLexicalReady","openDuckdb"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AACzD,SAASC,iBAAiB,EAAEC,cAAc,EAAEC,SAAS,EAAEC,eAAeC,cAAc,QAAQ,eAAe;AAC3G,SAASC,SAAS,QAAQ,eAAe;AAEzC,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,UAAU,EAAEC,YAAY,QAAQ,eAAe;AACxE,SAASC,cAAc,EAAEC,SAAS,EAAEC,YAAY,QAAQ,cAAc;AACtE,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,eAAeC,iBAAiBC,OAAqB,EAAEC,IAAgB;IACrE,MAAMA,KAAKC,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMD,KAAKC,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMD,KAAKC,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,MAAMD,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,IAAI,AAAC,MAAMlB,QAAQiB,MAAM,sBAAuB,MAAM,MAAMhB,QAAQgB,MAAM,kBAAkBH;AAC9F;AAEA,eAAeK,oBAAoBH,OAAqB,EAAEC,IAAgB,EAAEG,GAAW;IACrF,KAAK,MAAMC,WAAW7B,eAAe4B,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIC,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKC,IAAI,CAAC,CAAC,kKAAkK,EAAE5B,WAAW,+BAA+B,CAAC;YAChO;QACF;QACA,MAAM+B,QAAQE,MAAM,CAACN;IACvB;AACF;AAEA,eAAeO,kBAAkBC,MAAoB,EAAEC,KAAiB,EAAEC,IAAY;IACpF,MAAMf,uBAAuBa,OAAOG,MAAM,EAAEtC;AAC9C;AAEA,iFAAiF;AACjF,eAAeuC,MAAMJ,MAAoB;IACvCA,OAAOG,MAAM,CAACE,cAAc;IAC5BL,OAAOM,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEP,IAAoB,EAAEQ,OAA0D;IACrH,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAM7B,WAAU;IACxC,EAAE,OAAO8B,KAAK;QACZ,IAAIA,eAAe9C,YAAY,MAAM8C;QACrC,MAAM,IAAI9C,WAAW,4BAA4B,CAAC,yBAAyB,EAAEe,eAAe,UAAU,EAAE,AAAC+B,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIV;IACJ,IAAIG;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQC,QAAQK,aAAa,GAAG;YAAEC,aAAa;QAAY,IAAI;YAAEC,+BAA+B;QAAS;QAChJd,SAAS,MAAMG,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAIzC,WAAW,4BAA4B,CAAC,8BAA8B,EAAE2C,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAM5B,kBAAkBkB;QACxB,MAAMX,OAAOf,iBAAiB0B;QAC9B,OAAO;YAAEH,QAAQ;gBAAEM;gBAAUH;YAAO;YAAGX;QAAK;IAC9C,EAAE,OAAOoB,KAAK;QACZ,MAAMR,MAAM;YAAEE;YAAUH;QAAO;QAC/B,MAAMS;IACR;AACF;AAEA,eAAeM;IACb,OAAOjD,eAAeY,gBAAgB,AAAC,CAAA,MAAME,cAAa,EAAGoC,OAAO;AACtE;AAEA,4FAA4F;AAC5F,eAAeC;IACb,MAAM,EAAEC,GAAG,EAAEF,OAAO,EAAE,GAAG,MAAMpC;IAC/B,MAAMuB,WAAW,MAAMe,IAAIV,cAAc,CAACG,MAAM,CAAC;IACjD,MAAMd,SAAS;QAAEM;QAAUH,QAAQ,MAAMG,SAASE,OAAO;IAAG;IAC5D,IAAI;QACF,MAAMhB,OAAOf,iBAAiBuB,OAAOG,MAAM;QAC3C,MAAMX,KAAKC,IAAI,CAAC;QAChB,MAAMH,iBAAiBU,QAAQR;QAC/B,MAAMA,KAAK8B,QAAQ,CACjB,0EACAtD,kBAAkBuD,GAAG,CAAC,CAACC,MAAQ;gBAACA,IAAIC,IAAI;gBAAED,IAAIE,KAAK;gBAAEF,IAAIG,OAAO;gBAAEH,IAAII,IAAI;aAAC;QAE7E,MAAMxD,eAAe,UAAUH,eAAeY,gBAAgBsC,UAAUjD,UAAUW,gBAAgBR,UAAUQ,kBAAkB,OAAOgD;YACnI,MAAMlD,WAAWa;YACjB,OAAO,AAAC,CAAA,MAAMZ,aAAaY,MAAMqC,OAAO;gBAAEC,WAAW;gBAAIC,WAAW;gBAAIC,WAAW;gBAAIC,OAAO;YAAG,EAAC,EAAGV,GAAG,CAAC,CAACW,MAAQA,IAAIT,IAAI;QAC5H;IACF,SAAU;QACR,MAAMrB,MAAMJ;IACd;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMmC,oBAA+C;IAC1DC,UAAUhD;IACViD,eAAehD;IACfiD,kBAAkBtD;IAClBwB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFmC,UAAU,CAAC3B,MAAQ,2DAA2D4B,IAAI,CAAC5B,IAAIC,OAAO;IAC9F5C,gBAAgBiD;IAChB/C,aAAaiD;IACb9B;IACAI;IACAK;IACA0C,gBAAgB9D;IAChB+D,oBAAoBhE;IACpBQ,aAAa,CAACc,QAAQR,OAASN,YAAYc,OAAOM,QAAQ,EAAEN,OAAOG,MAAM,EAAEX;AAC7E,EAAE;AAEF,OAAO,eAAemD,WAAWhD,GAAmB,EAAEe,OAA6B;IACjF,OAAOpC,gBAAgBqB,KAAKwC,mBAAmBzB;AACjD"}

package/dist/esm/store/duckdb/ordered-bm25.d.ts

@@ -1,17 +1,4 @@
 import type { Connection } from '../types.js';
-export declare const ORDERED_BM25_MACRO = "sense_ordered_match_bm25";
-interface NativeMacroRow {
-    function_type: string;
-    parameters: unknown;
-    parameter_types: unknown;
-    macro_definition: string;
-}
-interface NativeMacroContract {
-    version: string;
-    macro: NativeMacroRow;
-}
-export declare function validateNativeMacroContract({ version, macro }: NativeMacroContract): string;
-export declare function ensureOrderedBm25(conn: Connection, state: {
-    orderedMacroReady: boolean;
-}): Promise<void>;
-export {};
+import type { DuckdbFtsBuilders } from './engine.js';
+export declare function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]>;
+export declare function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string;

package/dist/esm/store/duckdb/ordered-bm25.js

@@ -1,56 +1,15 @@
-import { createHash } from 'node:crypto';
-import { SenseError } from '../../errors.js';
-// This is a connection-local TEMP macro. The contract hash covers the native fts definition,
-// DuckDB version, and the reviewed parameter defaults captured in the retained diagnostic.
-export const ORDERED_BM25_MACRO = 'sense_ordered_match_bm25';
-const SUPPORTED_DUCKDB_VERSION = 'v1.5.5';
-const SUPPORTED_CONTRACT_SHA256 = '4476d543e43e7f8c89751cac0659b64e09d906cd3fefb1bfcaf4d15d3b1d9e00';
-const EXPECTED_PARAMETERS = [
-    'docname',
-    'query_string',
-    'fields',
-    'k',
-    'b',
-    'conjunctive'
-];
-const EXPECTED_DEFAULTS = {
-    fields: 'NULL',
-    k: '1.2',
-    b: '0.75',
-    conjunctive: 'false'
-};
-const EXPECTED_SIGNATURE = `docname, query_string, fields := ${EXPECTED_DEFAULTS.fields}, k := ${EXPECTED_DEFAULTS.k}, b := ${EXPECTED_DEFAULTS.b}, conjunctive := ${EXPECTED_DEFAULTS.conjunctive}`;
-const UNORDERED_SUM = 'sum(subscore)';
-function unsupported(message) {
-    return new SenseError('STORE_CAPABILITY_MISSING', `store "duckdb" cannot install its deterministic native BM25 adapter: ${message}; update the DuckDB adapter before using lexical search`);
-}
-function contractSha256(version, definition) {
-    return createHash('sha256').update(`${version}\0${EXPECTED_SIGNATURE}\0${definition}`).digest('hex');
-}
-export function validateNativeMacroContract({ version, macro }) {
-    if (version !== SUPPORTED_DUCKDB_VERSION) throw unsupported(`DuckDB ${version} is unsupported (validated version is ${SUPPORTED_DUCKDB_VERSION})`);
-    if (macro.function_type !== 'macro') throw unsupported(`match_bm25 is ${macro.function_type}, not a macro`);
-    if (JSON.stringify(macro.parameters) !== JSON.stringify(EXPECTED_PARAMETERS)) throw unsupported(`match_bm25 parameters changed (${JSON.stringify(macro.parameters)})`);
-    if (!Array.isArray(macro.parameter_types) || macro.parameter_types.some((type)=>type !== null)) throw unsupported('match_bm25 parameter types changed');
-    if (contractSha256(version, macro.macro_definition) !== SUPPORTED_CONTRACT_SHA256) throw unsupported('match_bm25 definition or reviewed parameter-default contract changed');
-    if (macro.macro_definition.split(UNORDERED_SUM).length !== 2) throw unsupported('match_bm25 does not contain exactly one validated unordered subscore sum');
-    return macro.macro_definition.replace(UNORDERED_SUM, 'sum(subscore ORDER BY subscore)');
-}
-export async function ensureOrderedBm25(conn, state) {
-    if (state.orderedMacroReady) return;
-    const versionStatement = await conn.prepare('SELECT version() AS version');
-    const versionRow = await versionStatement.get();
-    const macroStatement = await conn.prepare(`
-    SELECT function_type, parameters, parameter_types, macro_definition
-    FROM duckdb_functions()
-    WHERE schema_name = 'fts_main_content' AND function_name = 'match_bm25'
-  `);
-    const macro = await macroStatement.get();
-    if (!macro) throw unsupported('fts_main_content.match_bm25 is missing');
-    const orderedDefinition = validateNativeMacroContract({
-        version: versionRow.version,
-        macro
-    });
-    await conn.exec(`CREATE OR REPLACE TEMP MACRO ${ORDERED_BM25_MACRO}(${EXPECTED_SIGNATURE}) AS ${orderedDefinition}`);
-    state.orderedMacroReady = true;
+// Keep deduplication native, but return original tokens so native scoring stems once.
+export async function nativeQueryTerms(conn, fts, query) {
+    const stmt = await conn.prepare(fts.queryTerms());
+    const rows = await stmt.all(query);
+    return rows.map(({ term })=>term);
+}
+// Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.
+// Only the number of terms shapes this authored SQL; query values are always bound.
+export function orderedBm25Sql(fts, field, termCount) {
+    if (termCount === 0) return 'NULL';
+    const contributions = Array.from({
+        length: termCount
+    }, ()=>fts.termScore(field));
+    return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;
 }

package/dist/esm/store/duckdb/ordered-bm25.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { SenseError } from '../../errors.ts';\nimport type { Connection } from '../types.ts';\n\n// This is a connection-local TEMP macro. The contract hash covers the native fts definition,\n// DuckDB version, and the reviewed parameter defaults captured in the retained diagnostic.\nexport const ORDERED_BM25_MACRO = 'sense_ordered_match_bm25';\nconst SUPPORTED_DUCKDB_VERSION = 'v1.5.5';\nconst SUPPORTED_CONTRACT_SHA256 = '4476d543e43e7f8c89751cac0659b64e09d906cd3fefb1bfcaf4d15d3b1d9e00';\nconst EXPECTED_PARAMETERS = ['docname', 'query_string', 'fields', 'k', 'b', 'conjunctive'];\nconst EXPECTED_DEFAULTS = { fields: 'NULL', k: '1.2', b: '0.75', conjunctive: 'false' } as const;\nconst EXPECTED_SIGNATURE = `docname, query_string, fields := ${EXPECTED_DEFAULTS.fields}, k := ${EXPECTED_DEFAULTS.k}, b := ${EXPECTED_DEFAULTS.b}, conjunctive := ${EXPECTED_DEFAULTS.conjunctive}`;\nconst UNORDERED_SUM = 'sum(subscore)';\n\ninterface NativeMacroRow {\n  function_type: string;\n  parameters: unknown;\n  parameter_types: unknown;\n  macro_definition: string;\n}\n\ninterface NativeMacroContract {\n  version: string;\n  macro: NativeMacroRow;\n}\n\nfunction unsupported(message: string): SenseError {\n  return new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" cannot install its deterministic native BM25 adapter: ${message}; update the DuckDB adapter before using lexical search`);\n}\n\nfunction contractSha256(version: string, definition: string): string {\n  return createHash('sha256').update(`${version}\\0${EXPECTED_SIGNATURE}\\0${definition}`).digest('hex');\n}\n\nexport function validateNativeMacroContract({ version, macro }: NativeMacroContract): string {\n  if (version !== SUPPORTED_DUCKDB_VERSION) throw unsupported(`DuckDB ${version} is unsupported (validated version is ${SUPPORTED_DUCKDB_VERSION})`);\n  if (macro.function_type !== 'macro') throw unsupported(`match_bm25 is ${macro.function_type}, not a macro`);\n  if (JSON.stringify(macro.parameters) !== JSON.stringify(EXPECTED_PARAMETERS)) throw unsupported(`match_bm25 parameters changed (${JSON.stringify(macro.parameters)})`);\n  if (!Array.isArray(macro.parameter_types) || macro.parameter_types.some((type) => type !== null)) throw unsupported('match_bm25 parameter types changed');\n  if (contractSha256(version, macro.macro_definition) !== SUPPORTED_CONTRACT_SHA256) throw unsupported('match_bm25 definition or reviewed parameter-default contract changed');\n  if (macro.macro_definition.split(UNORDERED_SUM).length !== 2) throw unsupported('match_bm25 does not contain exactly one validated unordered subscore sum');\n  return macro.macro_definition.replace(UNORDERED_SUM, 'sum(subscore ORDER BY subscore)');\n}\n\nexport async function ensureOrderedBm25(conn: Connection, state: { orderedMacroReady: boolean }): Promise<void> {\n  if (state.orderedMacroReady) return;\n  const versionStatement = await conn.prepare('SELECT version() AS version');\n  const versionRow = (await versionStatement.get()) as { version: string };\n  const macroStatement = await conn.prepare(`\n    SELECT function_type, parameters, parameter_types, macro_definition\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content' AND function_name = 'match_bm25'\n  `);\n  const macro = (await macroStatement.get()) as NativeMacroRow | undefined;\n  if (!macro) throw unsupported('fts_main_content.match_bm25 is missing');\n  const orderedDefinition = validateNativeMacroContract({ version: versionRow.version, macro });\n  await conn.exec(`CREATE OR REPLACE TEMP MACRO ${ORDERED_BM25_MACRO}(${EXPECTED_SIGNATURE}) AS ${orderedDefinition}`);\n  state.orderedMacroReady = true;\n}\n"],"names":["createHash","SenseError","ORDERED_BM25_MACRO","SUPPORTED_DUCKDB_VERSION","SUPPORTED_CONTRACT_SHA256","EXPECTED_PARAMETERS","EXPECTED_DEFAULTS","fields","k","b","conjunctive","EXPECTED_SIGNATURE","UNORDERED_SUM","unsupported","message","contractSha256","version","definition","update","digest","validateNativeMacroContract","macro","function_type","JSON","stringify","parameters","Array","isArray","parameter_types","some","type","macro_definition","split","length","replace","ensureOrderedBm25","conn","state","orderedMacroReady","versionStatement","prepare","versionRow","get","macroStatement","orderedDefinition","exec"],"mappings":"AAAA,SAASA,UAAU,QAAQ,cAAc;AACzC,SAASC,UAAU,QAAQ,kBAAkB;AAG7C,6FAA6F;AAC7F,2FAA2F;AAC3F,OAAO,MAAMC,qBAAqB,2BAA2B;AAC7D,MAAMC,2BAA2B;AACjC,MAAMC,4BAA4B;AAClC,MAAMC,sBAAsB;IAAC;IAAW;IAAgB;IAAU;IAAK;IAAK;CAAc;AAC1F,MAAMC,oBAAoB;IAAEC,QAAQ;IAAQC,GAAG;IAAOC,GAAG;IAAQC,aAAa;AAAQ;AACtF,MAAMC,qBAAqB,CAAC,iCAAiC,EAAEL,kBAAkBC,MAAM,CAAC,OAAO,EAAED,kBAAkBE,CAAC,CAAC,OAAO,EAAEF,kBAAkBG,CAAC,CAAC,iBAAiB,EAAEH,kBAAkBI,WAAW,EAAE;AACpM,MAAME,gBAAgB;AActB,SAASC,YAAYC,OAAe;IAClC,OAAO,IAAIb,WAAW,4BAA4B,CAAC,qEAAqE,EAAEa,QAAQ,uDAAuD,CAAC;AAC5L;AAEA,SAASC,eAAeC,OAAe,EAAEC,UAAkB;IACzD,OAAOjB,WAAW,UAAUkB,MAAM,CAAC,GAAGF,QAAQ,EAAE,EAAEL,mBAAmB,EAAE,EAAEM,YAAY,EAAEE,MAAM,CAAC;AAChG;AAEA,OAAO,SAASC,4BAA4B,EAAEJ,OAAO,EAAEK,KAAK,EAAuB;IACjF,IAAIL,YAAYb,0BAA0B,MAAMU,YAAY,CAAC,OAAO,EAAEG,QAAQ,sCAAsC,EAAEb,yBAAyB,CAAC,CAAC;IACjJ,IAAIkB,MAAMC,aAAa,KAAK,SAAS,MAAMT,YAAY,CAAC,cAAc,EAAEQ,MAAMC,aAAa,CAAC,aAAa,CAAC;IAC1G,IAAIC,KAAKC,SAAS,CAACH,MAAMI,UAAU,MAAMF,KAAKC,SAAS,CAACnB,sBAAsB,MAAMQ,YAAY,CAAC,+BAA+B,EAAEU,KAAKC,SAAS,CAACH,MAAMI,UAAU,EAAE,CAAC,CAAC;IACrK,IAAI,CAACC,MAAMC,OAAO,CAACN,MAAMO,eAAe,KAAKP,MAAMO,eAAe,CAACC,IAAI,CAAC,CAACC,OAASA,SAAS,OAAO,MAAMjB,YAAY;IACpH,IAAIE,eAAeC,SAASK,MAAMU,gBAAgB,MAAM3B,2BAA2B,MAAMS,YAAY;IACrG,IAAIQ,MAAMU,gBAAgB,CAACC,KAAK,CAACpB,eAAeqB,MAAM,KAAK,GAAG,MAAMpB,YAAY;IAChF,OAAOQ,MAAMU,gBAAgB,CAACG,OAAO,CAACtB,eAAe;AACvD;AAEA,OAAO,eAAeuB,kBAAkBC,IAAgB,EAAEC,KAAqC;IAC7F,IAAIA,MAAMC,iBAAiB,EAAE;IAC7B,MAAMC,mBAAmB,MAAMH,KAAKI,OAAO,CAAC;IAC5C,MAAMC,aAAc,MAAMF,iBAAiBG,GAAG;IAC9C,MAAMC,iBAAiB,MAAMP,KAAKI,OAAO,CAAC,CAAC;;;;EAI3C,CAAC;IACD,MAAMnB,QAAS,MAAMsB,eAAeD,GAAG;IACvC,IAAI,CAACrB,OAAO,MAAMR,YAAY;IAC9B,MAAM+B,oBAAoBxB,4BAA4B;QAAEJ,SAASyB,WAAWzB,OAAO;QAAEK;IAAM;IAC3F,MAAMe,KAAKS,IAAI,CAAC,CAAC,6BAA6B,EAAE3C,mBAAmB,CAAC,EAAES,mBAAmB,KAAK,EAAEiC,mBAAmB;IACnHP,MAAMC,iBAAiB,GAAG;AAC5B"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import type { Connection } from '../types.ts';\nimport type { DuckdbFtsBuilders } from './engine.ts';\n\n// Keep deduplication native, but return original tokens so native scoring stems once.\nexport async function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]> {\n  const stmt = await conn.prepare(fts.queryTerms());\n  const rows = (await stmt.all(query)) as Array<{ term: string }>;\n  return rows.map(({ term }) => term);\n}\n\n// Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.\n// Only the number of terms shapes this authored SQL; query values are always bound.\nexport function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string {\n  if (termCount === 0) return 'NULL';\n  const contributions = Array.from({ length: termCount }, () => fts.termScore(field));\n  return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;\n}\n"],"names":["nativeQueryTerms","conn","fts","query","stmt","prepare","queryTerms","rows","all","map","term","orderedBm25Sql","field","termCount","contributions","Array","from","length","termScore","join"],"mappings":"AAGA,sFAAsF;AACtF,OAAO,eAAeA,iBAAiBC,IAAgB,EAAEC,GAAsB,EAAEC,KAAa;IAC5F,MAAMC,OAAO,MAAMH,KAAKI,OAAO,CAACH,IAAII,UAAU;IAC9C,MAAMC,OAAQ,MAAMH,KAAKI,GAAG,CAACL;IAC7B,OAAOI,KAAKE,GAAG,CAAC,CAAC,EAAEC,IAAI,EAAE,GAAKA;AAChC;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,SAASC,eAAeT,GAAsB,EAAEU,KAAmC,EAAEC,SAAiB;IAC3G,IAAIA,cAAc,GAAG,OAAO;IAC5B,MAAMC,gBAAgBC,MAAMC,IAAI,CAAC;QAAEC,QAAQJ;IAAU,GAAG,IAAMX,IAAIgB,SAAS,CAACN;IAC5E,OAAO,CAAC,wCAAwC,EAAEE,cAAcK,IAAI,CAAC,eAAe,EAAE,CAAC;AACzF"}