sensemaking 0.17.0 → 0.18.0

package/dist/esm/store/duckdb/lexical.js

@@ -0,0 +1,163 @@
+// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /
+// unspaced scripts, JS excerpts for every doc (no snippet()).
+import { SenseError } from '../../errors.js';
+import { UNSPACED_SCRIPTS } from '../../text/segment.js';
+import { withTransaction } from '../transaction.js';
+const FIELDS = [
+    'title',
+    'summary',
+    'text'
+];
+// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's
+// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored
+// separately and combined here.
+const FIELD_WEIGHT = {
+    title: 10,
+    summary: 5,
+    text: 1
+};
+const UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');
+// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would
+// otherwise silently treat as literal terms instead of honoring (PRINCIPLES.md #6). Checked
+// against terms with quoted spans blanked -- those go through contains() and are supported.
+const FTS5_OPERATORS = [
+    {
+        label: 'prefix query',
+        re: /[\p{L}\p{N}_]+\*(?=\s|$)/u
+    },
+    {
+        label: 'boolean operator',
+        re: /(?:^|\s)(?:AND|OR|NOT)(?=\s|$)/
+    },
+    {
+        label: 'NEAR operator',
+        re: /(?:^|\s)NEAR\b/
+    },
+    {
+        label: 'initial-token operator',
+        re: /(?:^|\s)\^\S+/
+    },
+    {
+        label: 'column filter',
+        re: /(?:^|\s)[\p{L}_]\w*\s*:/u
+    }
+];
+function unsupportedOperator(terms) {
+    const withoutPhrases = terms.replace(/"[^"]*"/g, ' ');
+    for (const { label, re } of FTS5_OPERATORS){
+        const m = withoutPhrases.match(re);
+        if (m) return {
+            label,
+            token: m[0].trim()
+        };
+    }
+    return null;
+}
+// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index
+// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --
+// and any author-quoted phrase, any script -- go through contains() instead of the fts index.
+function splitTerms(terms) {
+    const words = [];
+    const substrings = [];
+    const withoutPhrases = terms.replace(/"([^"]*)"/g, (_m, inner)=>{
+        const phrase = inner.trim();
+        if (phrase.length === 0) return ' ';
+        substrings.push(phrase);
+        if (!UNSPACED_RUN.test(phrase)) {
+            for (const w of phrase.split(/\s+/))if (w.length > 0) words.push(w);
+        }
+        return ' ';
+    });
+    for (const tok of withoutPhrases.split(/\s+/)){
+        if (tok.length === 0) continue;
+        if (UNSPACED_RUN.test(tok)) substrings.push(tok);
+        else words.push(tok);
+    }
+    return {
+        words,
+        substrings
+    };
+}
+// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this
+// pays the full rebuild -- but only once per store instance's first lexical query, and only
+// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).
+async function ensureFtsFresh(conn, state) {
+    if (!state.stale) return;
+    await conn.exec('INSTALL fts; LOAD fts;');
+    await withTransaction(conn, async ()=>{
+        // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either
+        // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise
+        // silently drop common query words (e.g. "and") from the index but not from match_bm25's
+        // conjunctive gate, making a bare multi-word query that contains one match nothing.
+        await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);
+    });
+    state.stale = false;
+}
+// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both
+// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are
+// emitted -- positional `?` binding requires that order to match the assembled SQL text.
+function buildScoreAndGate(words, substrings) {
+    const scoreParts = [];
+    const gateParts = [];
+    const params = [];
+    if (words.length > 0) {
+        const wordQuery = words.join(' ');
+        for (const field of FIELDS){
+            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
+            params.push(wordQuery);
+        }
+        gateParts.push(`fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`);
+        params.push(wordQuery);
+    }
+    for (const raw of substrings){
+        const needle = raw.toLowerCase();
+        for (const field of FIELDS){
+            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);
+            params.push(needle);
+        }
+        gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);
+        params.push(needle);
+    }
+    return {
+        scoreSql: scoreParts.join(' + '),
+        gateSql: gateParts.join(' AND '),
+        params
+    };
+}
+// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's
+// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes
+// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.
+export async function queryLexical(conn, terms, opts, state) {
+    const unsupported = unsupportedOperator(terms);
+    if (unsupported !== null) {
+        throw new SenseError('STORE_CAPABILITY_MISSING', `store "duckdb" does not implement FTS5's ${unsupported.label} ("${unsupported.token}") in this build; rephrase "${terms.trim()}" without it, or set "store" to "sqlite" in this tree's config to search it as written`);
+    }
+    const { whereJoin, whereCond, scopeCond, limit } = opts;
+    const { words, substrings } = splitTerms(terms);
+    if (words.length === 0 && substrings.length === 0) return [];
+    if (words.length > 0) await ensureFtsFresh(conn, state);
+    const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);
+    const sql = `SELECT path, NULL AS hit FROM (
+    SELECT content."path" AS path, (${scoreSql}) AS score
+    FROM content
+    ${whereJoin}
+    WHERE ${gateSql}
+    ${whereCond} ${scopeCond}
+  ) sq ORDER BY score DESC, path LIMIT ?`;
+    const stmt = await conn.prepare(sql);
+    return await stmt.all(...params, limit);
+}
+// One instance per store: `stale` starts true (a fresh connection cannot know whether an
+// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back
+// to true whenever content changes, so the next query rebuilds before ranking against it.
+export function createLexicalIndex(conn) {
+    const state = {
+        stale: true
+    };
+    return {
+        query: (terms, opts)=>queryLexical(conn, terms, opts, state),
+        markStale: ()=>{
+            state.stale = true;
+        }
+    };
+}

package/dist/esm/store/duckdb/lexical.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES.md #6). Checked\n// against terms with quoted spans blanked -- those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text.\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const params: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      params.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    params.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      params.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    params.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","UNSPACED_SCRIPTS","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","params","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAER,iBAAiB,CAAC,CAAC,EAAE;AAEzD,uFAAuF;AACvF,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMS,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAClB,aAAaqB,IAAI,CAACH,SAAS;YAAA,KAAK,MAAMI,KAAKJ,OAAOK,KAAK,CAAC,OAAQ,IAAID,EAAEH,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACE;QAAE;QACpG,OAAO;IACT;IACA,KAAK,MAAME,OAAOjB,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIL,MAAM,KAAK,GAAG;QACtB,IAAInB,aAAaqB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;aACvCV,MAAMM,IAAI,CAACI;IAClB;IACA,OAAO;QAAEV;QAAOC;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,eAAeU,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMnC,gBAAgBgC,MAAM;QAC1B,uFAAuF;QACvF,0FAA0F;QAC1F,yFAAyF;QACzF,oFAAoF;QACpF,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,yFAAyF;AACzF,SAASE,kBAAkBhB,KAAe,EAAEC,UAAoB;IAC9D,MAAMgB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,SAAoB,EAAE;IAE5B,IAAInB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMe,YAAYpB,MAAMqB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAASzC,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACwC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJH,OAAOb,IAAI,CAACc;QACd;QACAF,UAAUZ,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGa,OAAOb,IAAI,CAACc;IACd;IAEA,KAAK,MAAMG,OAAOtB,WAAY;QAC5B,MAAMuB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAASzC,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACwC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGH,OAAOb,IAAI,CAACkB;QACd;QACAN,UAAUZ,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHa,OAAOb,IAAI,CAACkB;IACd;IAEA,OAAO;QAAEE,UAAUT,WAAWI,IAAI,CAAC;QAAQM,SAAST,UAAUG,IAAI,CAAC;QAAUF;IAAO;AACtF;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,mGAAmG;AACnG,OAAO,eAAeS,aAAahB,IAAgB,EAAEpB,KAAa,EAAEqC,IAAyB,EAAEhB,KAAoB;IACjH,MAAMiB,cAAcvC,oBAAoBC;IACxC,IAAIsC,gBAAgB,MAAM;QACxB,MAAM,IAAIpD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEoD,YAAYzC,KAAK,CAAC,GAAG,EAAEyC,YAAYjC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEiC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE7B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMM,eAAeC,MAAMC;IAEjD,MAAM,EAAEa,QAAQ,EAAEC,OAAO,EAAER,MAAM,EAAE,GAAGH,kBAAkBhB,OAAOC;IAC/D,MAAMkC,MAAM,CAAC;oCACqB,EAAET,SAAS;;IAE3C,EAAEK,UAAU;UACN,EAAEJ,QAAQ;IAChB,EAAEK,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAMxB,KAAKyB,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAInB,QAAQe;AACpC;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,0FAA0F;AAC1F,OAAO,SAASK,mBAAmB3B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL0B,OAAO,CAAChD,OAAOqC,OAASD,aAAahB,MAAMpB,OAAOqC,MAAMhB;QACxD4B,WAAW;YACT5B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}

package/dist/esm/store/duckdb/native.d.ts

@@ -0,0 +1,3 @@
+export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
+export declare function loadOrInstall<T>(importName: string, nodeModulesPath: string): Promise<T>;
+export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/esm/store/duckdb/native.js

@@ -0,0 +1,96 @@
+import { existsSync, readFileSync } from 'node:fs';
+import Module from 'node:module';
+import { dirname, join } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { SenseError } from '../../errors.js';
+export const DUCKDB_PACKAGE = '@duckdb/node-api';
+// install-module-linked itself is small, but its target (@duckdb/node-api) is not: deferred the
+// same way embed/static.ts defers @huggingface/tokenizers, so requiring it stays inside the
+// already-gated branch below rather than running for every store/index.ts load.
+const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
+const _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));
+// Walks up from this file's own directory to find the real package.json, rather than a fixed
+// dot-count from __dirname (xz-compat's `../../node_modules`): `tsds test:node` runs straight
+// off src/, one directory shallower than dist/{cjs,esm}/store/duckdb sits under the built
+// package, so a hardcoded depth is right for one and wrong for the other. Works the same way
+// for a globally-installed `sense`, whose node_modules this resolves to -- never the caller's
+// cwd.
+function packageNodeModules() {
+    let dir = _dirname;
+    for(;;){
+        const candidate = join(dir, 'package.json');
+        if (existsSync(candidate)) {
+            const pkg = JSON.parse(readFileSync(candidate, 'utf8'));
+            if (pkg.name === 'sensemaking') return join(dir, 'node_modules');
+        }
+        const parent = dirname(dir);
+        if (parent === dir) throw new Error('could not locate the sensemaking package root to install @duckdb/node-api into');
+        dir = parent;
+    }
+}
+const installing = new Map();
+// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).
+// The install here is async -- the load site already awaits a dynamic import -- so a bare
+// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target
+// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,
+// keyed by specifier and target so tests exercising several installs in one process don't share
+// one slot. A failed install is not cached, so a later call retries.
+function installOnce(specifier, nodeModulesPath) {
+    const key = `${specifier}\n${nodeModulesPath}`;
+    let promise = installing.get(key);
+    if (!promise) {
+        const installModuleLinked = _require('install-module-linked').default;
+        promise = installModuleLinked(specifier, nodeModulesPath).then(()=>{}, (err)=>{
+            installing.delete(key);
+            throw err;
+        });
+        installing.set(key, promise);
+    }
+    return promise;
+}
+// Node caches a bare specifier's package resolution -- including a "not found" miss -- for the
+// life of the process, so retrying `import(importName)` right after installing sees the same
+// stale miss and falls through to a bogus index.js guess instead of the package's real entry
+// (verified against a real global install: the plain-specifier retry failed there, this did
+// not). Reading the freshly-installed package's own package.json and importing its entry file
+// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.
+function resolveEntryUrl(importName, nodeModulesPath) {
+    var _pkg_main;
+    const pkgDir = join(nodeModulesPath, ...importName.split('/'));
+    const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8'));
+    const entry = typeof pkg.exports === 'string' ? pkg.exports : (_pkg_main = pkg.main) !== null && _pkg_main !== void 0 ? _pkg_main : 'index.js';
+    return pathToFileURL(join(pkgDir, entry)).href;
+}
+// Import-then-install-then-retry, over an injectable target so the failure path (import AND
+// install both fail) can be exercised for real in tests without downloading @duckdb/node-api's
+// real ~110MB native binding. Production always passes @duckdb/node-api's own name; the thrown
+// error names @duckdb/node-api regardless, since that is what a real user needs to install.
+export async function loadOrInstall(importName, nodeModulesPath) {
+    try {
+        return await import(importName);
+    } catch  {
+        console.error(`sense: store "duckdb" needs ${DUCKDB_PACKAGE}; installing it now (one-time download, ~110MB)...`);
+        try {
+            await installOnce(importName, nodeModulesPath);
+        } catch (installErr) {
+            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "duckdb" needs ${DUCKDB_PACKAGE}, and installing it automatically failed (${installErr.message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \`npm install ${DUCKDB_PACKAGE}\` yourself and try again`);
+        }
+        try {
+            return await import(resolveEntryUrl(importName, nodeModulesPath));
+        } catch (loadErr) {
+            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "duckdb" installed ${DUCKDB_PACKAGE} but it could not be loaded (${loadErr.message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \`npm install ${DUCKDB_PACKAGE}\` to see the underlying error`);
+        }
+    }
+}
+let duckdbApiPromise;
+// The one accessor every consumer of @duckdb/node-api (open.ts, sql-functions.ts, vectors.ts)
+// must go through, rather than each doing its own `import('@duckdb/node-api')`: once a fresh
+// install has happened in this process, Node keeps returning that pre-install "not found" miss
+// to any NEW bare-specifier lookup of the same path, so a second independent import elsewhere
+// in the process fails even though the package is now genuinely on disk (the same stale-cache
+// problem loadOrInstall works around for its own retry). Caching the resolved module here means
+// only the first caller ever resolves the bare specifier at all.
+export function duckdbApi() {
+    if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall(DUCKDB_PACKAGE, packageNodeModules());
+    return duckdbApiPromise;
+}

package/dist/esm/store/duckdb/native.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../../errors.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\n// install-module-linked itself is small, but its target (@duckdb/node-api) is not: deferred the\n// same way embed/static.ts defers @huggingface/tokenizers, so requiring it stays inside the\n// already-gated branch below rather than running for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory to find the real package.json, rather than a fixed\n// dot-count from __dirname (xz-compat's `../../node_modules`): `tsds test:node` runs straight\n// off src/, one directory shallower than dist/{cjs,esm}/store/duckdb sits under the built\n// package, so a hardcoded depth is right for one and wrong for the other. Works the same way\n// for a globally-installed `sense`, whose node_modules this resolves to -- never the caller's\n// cwd.\nfunction packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install @duckdb/node-api into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).\n// The install here is async -- the load site already awaits a dynamic import -- so a bare\n// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target\n// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,\n// keyed by specifier and target so tests exercising several installs in one process don't share\n// one slot. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's package resolution -- including a \"not found\" miss -- for the\n// life of the process, so retrying `import(importName)` right after installing sees the same\n// stale miss and falls through to a bogus index.js guess instead of the package's real entry\n// (verified against a real global install: the plain-specifier retry failed there, this did\n// not). Reading the freshly-installed package's own package.json and importing its entry file\n// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable target so the failure path (import AND\n// install both fail) can be exercised for real in tests without downloading @duckdb/node-api's\n// real ~110MB native binding. Production always passes @duckdb/node-api's own name; the thrown\n// error names @duckdb/node-api regardless, since that is what a real user needs to install.\nexport async function loadOrInstall<T>(importName: string, nodeModulesPath: string): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"duckdb\" needs ${DUCKDB_PACKAGE}; installing it now (one-time download, ~110MB)...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"duckdb\" needs ${DUCKDB_PACKAGE}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${DUCKDB_PACKAGE}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" installed ${DUCKDB_PACKAGE} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${DUCKDB_PACKAGE}\\` to see the underlying error`);\n    }\n  }\n}\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api (open.ts, sql-functions.ts, vectors.ts)\n// must go through, rather than each doing its own `import('@duckdb/node-api')`: once a fresh\n// install has happened in this process, Node keeps returning that pre-install \"not found\" miss\n// to any NEW bare-specifier lookup of the same path, so a second independent import elsewhere\n// in the process fails even though the package is now genuinely on disk (the same stale-cache\n// problem loadOrInstall works around for its own retry). Caching the resolved module here means\n// only the first caller ever resolves the bare specifier at all.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB_PACKAGE, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","DUCKDB_PACKAGE","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","console","error","installErr","message","loadErr","process","platform","arch","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAE7C,OAAO,MAAMC,iBAAiB,mBAAmB;AAEjD,gGAAgG;AAChG,4FAA4F;AAC5F,gFAAgF;AAChF,MAAMC,WAAW,OAAOC,YAAY,cAAcR,OAAOS,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWV,QAAQ,OAAOW,eAAe,cAAcA,aAAaT,cAAc,YAAYO,GAAG;AAEvG,6FAA6F;AAC7F,8FAA8F;AAC9F,0FAA0F;AAC1F,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO;AACP,SAASG;IACP,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYb,KAAKY,KAAK;QAC5B,IAAIhB,WAAWiB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAACnB,aAAagB,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOjB,KAAKY,KAAK;QACnD;QACA,MAAMM,SAASnB,QAAQa;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,qEAAqE;AACrE,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,6FAA6F;AAC7F,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASpC,KAAKwB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAACnB,aAAaG,KAAKoC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOZ,cAAcF,KAAKoC,QAAQE,QAAQG,IAAI;AAChD;AAEA,4FAA4F;AAC5F,+FAA+F;AAC/F,+FAA+F;AAC/F,4FAA4F;AAC5F,OAAO,eAAeC,cAAiBP,UAAkB,EAAEX,eAAuB;IAChF,IAAI;QACF,OAAQ,MAAM,MAAM,CAACW;IACvB,EAAE,OAAM;QACNQ,QAAQC,KAAK,CAAC,CAAC,4BAA4B,EAAExC,eAAe,kDAAkD,CAAC;QAC/G,IAAI;YACF,MAAMkB,YAAYa,YAAYX;QAChC,EAAE,OAAOqB,YAAY;YACnB,MAAM,IAAI1C,WACR,4BACA,CAAC,qBAAqB,EAAEC,eAAe,0CAA0C,EAAE,AAACyC,WAAqBC,OAAO,CAAC,sKAAsK,EAAE1C,eAAe,yBAAyB,CAAC;QAEtU;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAAC8B,gBAAgBC,YAAYX;QACnD,EAAE,OAAOuB,SAAS;YAChB,MAAM,IAAI5C,WAAW,4BAA4B,CAAC,yBAAyB,EAAEC,eAAe,6BAA6B,EAAE,AAAC2C,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAE9C,eAAe,8BAA8B,CAAC;QAChT;IACF;AACF;AAEA,IAAI+C;AAEJ,8FAA8F;AAC9F,6FAA6F;AAC7F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,gGAAgG;AAChG,iEAAiE;AACjE,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBT,cAAiDtC,gBAAgBO;IAC3G,OAAOwC;AACT"}

package/dist/esm/store/duckdb/open.d.ts

@@ -0,0 +1,12 @@
+import type { ResolvedConfig } from '../../config/index.js';
+import type { Store } from '../types.js';
+export declare const DB_FILENAME = "cache.duckdb";
+export declare const SCHEMA_VERSION = "2";
+export interface OpenResult {
+    store: Store;
+    cfg: ResolvedConfig;
+    dbPath: string;
+    parsed: number;
+    warnings: string[];
+}
+export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/esm/store/duckdb/open.js

@@ -0,0 +1,127 @@
+import { mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { featureSignature, STATE_DIR } from '../../config/index.js';
+import { STORE_DIMS } from '../../embed/types.js';
+import { SenseError } from '../../errors.js';
+import { activeFeatures, FEATURES } from '../../features/index.js';
+import { getMeta, setMeta } from '../shared.js';
+import { clearCache } from '../sqlite/open.js';
+import { createConnection } from './connection.js';
+import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
+import { reconcile } from './reconcile.js';
+import { registerFunctions } from './sql-functions.js';
+import { createStore } from './store.js';
+export const DB_FILENAME = 'cache.duckdb';
+// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately
+// (VARIANT frontmatter columns instead of untyped ones), and the store name already joins the
+// feature signature (see featureSignature), so a config's `store` key switching cleanly
+// rebuilds rather than reusing the other engine's stale cache shape.
+export const SCHEMA_VERSION = '2';
+// `content` is a plain table (not FTS-virtual): D1 builds DuckDB's fts index over it lazily,
+// only when a lexical query actually runs (see duckdb/lexical.ts), and reads it directly for
+// contains() substring/phrase/unspaced-script verification either way. No tokenizer resolution
+// -- this store always uses the fts extension's default (porter) stemmer, unlike sqlite's
+// configurable `content.tokenize` (not read here; D1 does not extend that knob to DuckDB).
+async function ensureSchema(conn, cfg) {
+    await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
+    await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
+    await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
+    await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
+    for (const feature of activeFeatures(cfg)){
+        // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL
+        // (see duckdb/vectors.ts). `scale` is kept, unused, so the feature's shared reconcile-time
+        // INSERT/DELETE (src/features/embed.ts) still names a column that exists on both stores.
+        if (feature.name === 'embed') {
+            await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY ("path", chunk))`);
+            continue;
+        }
+        await feature.schema(conn);
+    }
+    if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
+    if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
+}
+async function connect(cfg) {
+    const stateDir = join(cfg.baseDir, STATE_DIR);
+    mkdirSync(stateDir, {
+        recursive: true
+    });
+    const dbPath = join(stateDir, DB_FILENAME);
+    // Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this
+    // optional peer dependency, so nothing in this store's module graph imports it as a value
+    // until a duckdb tree is actually opened (types-only imports elsewhere are erased and cost
+    // nothing either way). Installed on first use if missing, and shared with sql-functions.ts
+    // and vectors.ts (see native.ts's duckdbApi).
+    let DuckDBInstance;
+    try {
+        ({ DuckDBInstance } = await duckdbApi());
+    } catch (err) {
+        if (err instanceof SenseError) throw err;
+        throw new SenseError('STORE_DEPENDENCY_MISSING', `store "duckdb" needs the ${DUCKDB_PACKAGE} package (${err.message})`);
+    }
+    let duckdb;
+    let instance;
+    try {
+        // On-disk files default to an older storage format for cross-version compatibility, which
+        // rejects VARIANT columns ("VARIANT columns are not supported in storage versions prior to
+        // v1.5.0"); this store's dynamic frontmatter columns need VARIANT (see reconcile.ts), so
+        // the format floor is pinned explicitly. :memory: databases are unaffected either way.
+        instance = await DuckDBInstance.create(dbPath, {
+            storage_compatibility_version: 'v1.5.0'
+        });
+        duckdb = await instance.connect();
+    } catch (err) {
+        // create() may have succeeded before connect() failed: close it, or its WAL stays open.
+        instance === null || instance === void 0 ? void 0 : instance.closeSync();
+        throw new SenseError('STORE_DEPENDENCY_MISSING', `store "duckdb" failed to open ${dbPath}: ${err.message}`);
+    }
+    // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL
+    // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.
+    let closed = false;
+    try {
+        await registerFunctions(duckdb);
+        const conn = createConnection(duckdb);
+        await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
+        const version = await getMeta(conn, 'schema_version');
+        const features = await getMeta(conn, 'features');
+        const wantFeatures = featureSignature(cfg, FEATURES);
+        if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
+            // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
+            // rebuild instead, same as sqlite's open() (see its comment for why this is announced).
+            const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
+            console.error(`sense: ${reason}; rebuilding the index`);
+            // Close before clearCache deletes the files underneath the still-open instance.
+            closed = true;
+            duckdb.disconnectSync();
+            instance.closeSync();
+            clearCache(cfg);
+            return connect(cfg);
+        }
+        await ensureSchema(conn, cfg);
+        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);
+        return {
+            instance,
+            duckdb,
+            conn,
+            cfg,
+            dbPath,
+            parsed,
+            warnings
+        };
+    } catch (err) {
+        if (!closed) {
+            duckdb.disconnectSync();
+            instance === null || instance === void 0 ? void 0 : instance.closeSync();
+        }
+        throw err;
+    }
+}
+export async function openDuckdb(cfg) {
+    const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
+    return {
+        store: createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir),
+        cfg: resolvedCfg,
+        dbPath,
+        parsed,
+        warnings
+    };
+}

package/dist/esm/store/duckdb/open.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { clearCache } from '../sqlite/open.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately\n// (VARIANT frontmatter columns instead of untyped ones), and the store name already joins the\n// feature signature (see featureSignature), so a config's `store` key switching cleanly\n// rebuilds rather than reusing the other engine's stale cache shape.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): D1 builds DuckDB's fts index over it lazily,\n// only when a lexical query actually runs (see duckdb/lexical.ts), and reads it directly for\n// contains() substring/phrase/unspaced-script verification either way. No tokenizer resolution\n// -- this store always uses the fts extension's default (porter) stemmer, unlike sqlite's\n// configurable `content.tokenize` (not read here; D1 does not extend that knob to DuckDB).\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL\n    // (see duckdb/vectors.ts). `scale` is kept, unused, so the feature's shared reconcile-time\n    // INSERT/DELETE (src/features/embed.ts) still names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this\n  // optional peer dependency, so nothing in this store's module graph imports it as a value\n  // until a duckdb tree is actually opened (types-only imports elsewhere are erased and cost\n  // nothing either way). Installed on first use if missing, and shared with sql-functions.ts\n  // and vectors.ts (see native.ts's duckdbApi).\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which\n    // rejects VARIANT columns (\"VARIANT columns are not supported in storage versions prior to\n    // v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (see reconcile.ts), so\n    // the format floor is pinned explicitly. :memory: databases are unaffected either way.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","STORE_DIMS","SenseError","activeFeatures","FEATURES","getMeta","setMeta","clearCache","createConnection","DUCKDB_PACKAGE","duckdbApi","reconcile","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,2FAA2F;AAC3F,8FAA8F;AAC9F,wFAAwF;AACxF,qEAAqE;AACrE,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,6FAA6F;AAC7F,6FAA6F;AAC7F,+FAA+F;AAC/F,0FAA0F;AAC1F,2FAA2F;AAC3F,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWjB,eAAee,KAAM;QACzC,wFAAwF;QACxF,2FAA2F;QAC3F,yFAAyF;QACzF,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAElB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMmB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMV,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKd;AACxG;AAEA,eAAemB,QAAQL,GAAmB;IACxC,MAAMM,WAAW1B,KAAKoB,IAAIO,OAAO,EAAEzB;IACnCH,UAAU2B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS7B,KAAK0B,UAAUV;IAE9B,wFAAwF;IACxF,0FAA0F;IAC1F,2FAA2F;IAC3F,2FAA2F;IAC3F,8CAA8C;IAC9C,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe3B,YAAY,MAAM2B;QACrC,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,0FAA0F;QAC1F,2FAA2F;QAC3F,yFAAyF;QACzF,uFAAuF;QACvFA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEyB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAMhC,QAAQY,MAAM;QACpC,MAAMqB,WAAW,MAAMjC,QAAQY,MAAM;QACrC,MAAMsB,eAAexC,iBAAiBmB,KAAKd;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB5B,WAAWW;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUM,MAAMC,KAAKA,IAAIO,OAAO;QACnE,OAAO;YAAEO;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,OAAO;QAAE8B,OAAOnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;QAAGP,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpI"}

package/dist/esm/store/duckdb/reconcile.d.ts

@@ -0,0 +1,6 @@
+import type { Config } from '../../config/index.js';
+import type { Connection } from '../types.js';
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
+    parsed: number;
+    warnings: string[];
+}>;

package/dist/esm/store/duckdb/reconcile.js

@@ -0,0 +1,141 @@
+import { SenseError } from '../../errors.js';
+import { activeFeatures } from '../../features/index.js';
+import { progress } from '../../output/progress.js';
+import { listFiles, RESERVED_COLUMNS } from '../../scan/index.js';
+import { reparseFiles } from '../../scan/reparse.js';
+import { getColumns, quoteIdent } from '../shared.js';
+import { withTransaction } from '../transaction.js';
+// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per
+// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain
+// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from
+// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one
+// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared
+// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the
+// only DuckDB type that can hold the different JS types mapValue() produces across files for
+// the same key).
+const CORE_FRONTMATTER_COLUMNS = new Set([
+    'path',
+    '_mtime',
+    '_ctime',
+    '_size',
+    '_parse_error'
+]);
+// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):
+// `path` is content's own primary key, so this is a plain per-doc row.
+const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)`;
+function contentRow(doc) {
+    return [
+        doc.relPath,
+        doc.search.title,
+        doc.search.summary,
+        doc.search.text
+    ];
+}
+// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity
+// fence anyway so a runaway frontmatter generator fails with a clear message instead of an
+// unbounded ALTER TABLE loop.
+const MAX_FRONTMATTER_COLUMNS = 10000;
+export async function reconcile(conn, cfg, baseDir) {
+    const files = listFiles(cfg, baseDir);
+    const currentSet = new Set(files.map((f)=>f.relPath));
+    const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
+    const existingRows = await existingStmt.all();
+    const existing = new Map(existingRows.map((r)=>[
+            r.path,
+            r
+        ]));
+    const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
+    const toReparse = files.filter((f)=>{
+        const row = existing.get(f.relPath);
+        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
+    });
+    if (vanished.length === 0 && toReparse.length === 0) return {
+        parsed: 0,
+        warnings: []
+    };
+    const features = activeFeatures(cfg);
+    const seenColumns = await getColumns(conn);
+    const report = progress('reparsing files', toReparse.length);
+    const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    report.finish();
+    for (const col of newColumns)seenColumns.add(col);
+    const allColumns = [
+        ...seenColumns
+    ];
+    if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {
+        throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
+    }
+    const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
+    const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
+    const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
+    const delta = {
+        files,
+        reparsed: parsedDocs.map((d)=>d.relPath),
+        added,
+        vanished
+    };
+    const addedSet = new Set(added);
+    const reparsedExisting = parsedDocs.map((d)=>d.relPath).filter((p)=>!addedSet.has(p));
+    await withTransaction(conn, async ()=>{
+        var _feature_remove, _feature_afterReconcile;
+        // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold
+        // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.
+        for (const col of newColumns)await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);
+        if (parsedDocs.length > 0) {
+            const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
+                    var _doc_data_col;
+                    if (col === 'path') return doc.relPath;
+                    if (col === '_mtime') return doc.mtimeMs;
+                    if (col === '_ctime') return doc.ctimeMs;
+                    if (col === '_size') return doc.size;
+                    if (col === '_parse_error') return doc.parseError;
+                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
+                }));
+            await conn.runBatch(insertSql, rows);
+        }
+        const contentTouched = [
+            ...vanished,
+            ...reparsedExisting
+        ];
+        if (contentTouched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', contentTouched.map((p)=>[
+                p
+            ]));
+        if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));
+        if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
+                p
+            ]));
+        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
+        // new docs have nothing to clear, which keeps cold builds linear.
+        const presetTouched = [
+            ...vanished,
+            ...reparsedExisting
+        ];
+        if (presetTouched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', presetTouched.map((p)=>[
+                p
+            ]));
+        const presetRows = [];
+        for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
+            doc.relPath,
+            presetName
+        ]);
+        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
+        const removedPaths = [
+            ...vanished,
+            ...reparsedExisting
+        ];
+        if (removedPaths.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, removedPaths, delta));
+        for (const feature of features){
+            var _feature_store;
+            const docsForFeature = parsedDocs.map((doc)=>({
+                    path: doc.relPath,
+                    extracted: doc.extracted[feature.name]
+                }));
+            await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
+        }
+        for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
+    });
+    return {
+        parsed: parsedDocs.length,
+        warnings
+    };
+}