sensemaking 0.18.4 → 0.19.0

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).\n// Checked against terms with quoted spans blanked -- those go through contains() and are\n// supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text. Score\n// and gate params are collected apart and joined at the end because the assembled SQL puts\n// every score part ahead of every gate part: appending to one array term by term matches that\n// text only while the query has words or substrings, never both, and silently shifts each `?`\n// by one as soon as it has both (PRINCIPLES: no-silent-modes -- a shifted bind answers a\n// different question rather than failing).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","UNSPACED_SCRIPTS","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAER,iBAAiB,CAAC,CAAC,EAAE;AAEzD,uFAAuF;AACvF,+FAA+F;AAC/F,yFAAyF;AACzF,aAAa;AACb,MAAMS,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAClB,aAAaqB,IAAI,CAACH,SAAS;YAAA,KAAK,MAAMI,KAAKJ,OAAOK,KAAK,CAAC,OAAQ,IAAID,EAAEH,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACE;QAAE;QACpG,OAAO;IACT;IACA,KAAK,MAAME,OAAOjB,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIL,MAAM,KAAK,GAAG;QACtB,IAAInB,aAAaqB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;aACvCV,MAAMM,IAAI,CAACI;IAClB;IACA,OAAO;QAAEV;QAAOC;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,eAAeU,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMnC,gBAAgBgC,MAAM;QAC1B,uFAAuF;QACvF,0FAA0F;QAC1F,yFAAyF;QACzF,oFAAoF;QACpF,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,+FAA+F;AAC/F,2FAA2F;AAC3F,8FAA8F;AAC9F,8FAA8F;AAC9F,yFAAyF;AACzF,2CAA2C;AAC3C,SAASE,kBAAkBhB,KAAe,EAAEC,UAAoB;IAC9D,MAAMgB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIpB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMgB,YAAYrB,MAAMsB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAAS1C,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACyC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYb,IAAI,CAACe;QACnB;QACAH,UAAUZ,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGc,WAAWd,IAAI,CAACe;IAClB;IAEA,KAAK,MAAMG,OAAOvB,WAAY;QAC5B,MAAMwB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAAS1C,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACyC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYb,IAAI,CAACmB;QACnB;QACAP,UAAUZ,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHc,WAAWd,IAAI,CAACmB;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,mGAAmG;AACnG,OAAO,eAAeU,aAAalB,IAAgB,EAAEpB,KAAa,EAAEuC,IAAyB,EAAElB,KAAoB;IACjH,MAAMmB,cAAczC,oBAAoBC;IACxC,IAAIwC,gBAAgB,MAAM;QACxB,MAAM,IAAItD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEsD,YAAY3C,KAAK,CAAC,GAAG,EAAE2C,YAAYnC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEmC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE/B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMM,eAAeC,MAAMC;IAEjD,MAAM,EAAEc,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBhB,OAAOC;IAC/D,MAAMoC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAM1B,KAAK2B,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,0FAA0F;AAC1F,OAAO,SAASK,mBAAmB7B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,CAAClD,OAAOuC,OAASD,aAAalB,MAAMpB,OAAOuC,MAAMlB;QACxD8B,WAAW;YACT9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),\n// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","hasUnspacedRun","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACvB,eAAeuB,SAAS;YAAA,KAAK,MAAMG,KAAKH,OAAOI,KAAK,CAAC,OAAQ,IAAID,EAAEF,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACC;QAAE;QACjG,OAAO;IACT;IACA,KAAK,MAAME,OAAOhB,eAAee,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACtB,IAAIxB,eAAe4B,MAAMR,WAAWK,IAAI,CAACG;aACpCT,MAAMM,IAAI,CAACG;IAClB;IACA,OAAO;QAAET;QAAOC;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,eAAeS,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMhC,gBAAgB6B,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASE,kBAAkBf,KAAe,EAAEC,UAAoB;IAC9D,MAAMe,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAInB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMe,YAAYpB,MAAMqB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAASvC,OAAQ;YAC1BiC,WAAWV,IAAI,CAAC,GAAGtB,YAAY,CAACsC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYZ,IAAI,CAACc;QACnB;QACAH,UAAUX,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGa,WAAWb,IAAI,CAACc;IAClB;IAEA,KAAK,MAAMG,OAAOtB,WAAY;QAC5B,MAAMuB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAASvC,OAAQ;YAC1BiC,WAAWV,IAAI,CAAC,GAAGtB,YAAY,CAACsC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYZ,IAAI,CAACkB;QACnB;QACAP,UAAUX,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHa,WAAWb,IAAI,CAACkB;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,kHAAkH;AAClH,yIAAyI;AACzI,OAAO,eAAeU,aAAalB,IAAgB,EAAEnB,KAAa,EAAEsC,IAAyB,EAAElB,KAAoB;IACjH,MAAMmB,cAAcxC,oBAAoBC;IACxC,IAAIuC,gBAAgB,MAAM;QACxB,MAAM,IAAInD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEmD,YAAY1C,KAAK,CAAC,GAAG,EAAE0C,YAAYlC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEkC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE9B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMK,eAAeC,MAAMC;IAEjD,MAAM,EAAEc,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBf,OAAOC;IAC/D,MAAMmC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAM1B,KAAK2B,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,+HAA+H;AAC/H,qHAAqH;AACrH,OAAO,SAASK,mBAAmB7B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,CAACjD,OAAOsC,OAASD,aAAalB,MAAMnB,OAAOsC,MAAMlB;QACxD8B,WAAW;YACT9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}

package/dist/esm/store/duckdb/native.js

@@ -6,13 +6,8 @@ const DUCKDB = {
     sizeHint: '~110MB'
 };
 let duckdbApiPromise;
-// The one accessor every consumer of @duckdb/node-api (open.ts, sql-functions.ts, vectors.ts)
-// must go through, rather than each doing its own `import('@duckdb/node-api')`: once a fresh
-// install has happened in this process, Node keeps returning that pre-install "not found" miss
-// to any NEW bare-specifier lookup of the same path, so a second independent import elsewhere
-// in the process fails even though the package is now genuinely on disk (the same stale-cache
-// problem loadOrInstall works around for its own retry). Caching the resolved module here means
-// only the first caller ever resolves the bare specifier at all.
+// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning
+// the pre-install "not found" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.
 export function duckdbApi() {
     if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall(DUCKDB, packageNodeModules());
     return duckdbApiPromise;

package/dist/esm/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\nconst DUCKDB: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB' };\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api (open.ts, sql-functions.ts, vectors.ts)\n// must go through, rather than each doing its own `import('@duckdb/node-api')`: once a fresh\n// install has happened in this process, Node keeps returning that pre-install \"not found\" miss\n// to any NEW bare-specifier lookup of the same path, so a second independent import elsewhere\n// in the process fails even though the package is now genuinely on disk (the same stale-cache\n// problem loadOrInstall works around for its own retry). Caching the resolved module here means\n// only the first caller ever resolves the bare specifier at all.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB","store","pkg","sizeHint","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AAEjD,MAAMC,SAA2B;IAAEC,OAAO;IAAUC,KAAKH;IAAgBI,UAAU;AAAS;AAE5F,IAAIC;AAEJ,8FAA8F;AAC9F,6FAA6F;AAC7F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,gGAAgG;AAChG,iEAAiE;AACjE,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBP,cAAiDG,QAAQF;IACnG,OAAOM;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\nconst DUCKDB: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB' };\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB","store","pkg","sizeHint","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AAEjD,MAAMC,SAA2B;IAAEC,OAAO;IAAUC,KAAKH;IAAgBI,UAAU;AAAS;AAE5F,IAAIC;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBP,cAAiDG,QAAQF;IACnG,OAAOM;AACT"}

package/dist/esm/store/duckdb/open.js

@@ -1,36 +1,31 @@
 import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { featureSignature, STATE_DIR } from '../../config/index.js';
+import { rekeyChunkText } from '../../embed/handoff.js';
 import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
+import { clearCache } from '../cache.js';
 import { getMeta, setMeta } from '../shared.js';
-import { clearCache } from '../sqlite/open.js';
 import { createConnection } from './connection.js';
 import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
 import { reconcile } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.duckdb';
-// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately
-// (VARIANT frontmatter columns instead of untyped ones), and the store name already joins the
-// feature signature (see featureSignature), so a config's `store` key switching cleanly
-// rebuilds rather than reusing the other engine's stale cache shape.
+// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
+// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
 export const SCHEMA_VERSION = '2';
-// `content` is a plain table (not FTS-virtual): D1 builds DuckDB's fts index over it lazily,
-// only when a lexical query actually runs (see duckdb/lexical.ts), and reads it directly for
-// contains() substring/phrase/unspaced-script verification either way. No tokenizer resolution
-// -- this store always uses the fts extension's default (porter) stemmer, unlike sqlite's
-// configurable `content.tokenize` (not read here; D1 does not extend that knob to DuckDB).
+// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
+// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.
 async function ensureSchema(conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
     for (const feature of activeFeatures(cfg)){
-        // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL
-        // (see duckdb/vectors.ts). `scale` is kept, unused, so the feature's shared reconcile-time
-        // INSERT/DELETE (src/features/embed.ts) still names a column that exists on both stores.
+        // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,
+        // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.
         if (feature.name === 'embed') {
             await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY ("path", chunk))`);
             continue;
@@ -46,11 +41,8 @@ async function connect(cfg) {
         recursive: true
     });
     const dbPath = join(stateDir, DB_FILENAME);
-    // Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this
-    // optional peer dependency, so nothing in this store's module graph imports it as a value
-    // until a duckdb tree is actually opened (types-only imports elsewhere are erased and cost
-    // nothing either way). Installed on first use if missing, and shared with sql-functions.ts
-    // and vectors.ts (see native.ts's duckdbApi).
+    // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports
+    // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.
     let DuckDBInstance;
     try {
         ({ DuckDBInstance } = await duckdbApi());
@@ -61,10 +53,8 @@ async function connect(cfg) {
     let duckdb;
     let instance;
     try {
-        // On-disk files default to an older storage format for cross-version compatibility, which
-        // rejects VARIANT columns ("VARIANT columns are not supported in storage versions prior to
-        // v1.5.0"); this store's dynamic frontmatter columns need VARIANT (see reconcile.ts), so
-        // the format floor is pinned explicitly. :memory: databases are unaffected either way.
+        // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns ("VARIANT
+        // columns are not supported in storage versions prior to v1.5.0"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.
         instance = await DuckDBInstance.create(dbPath, {
             storage_compatibility_version: 'v1.5.0'
         });
@@ -117,8 +107,11 @@ async function connect(cfg) {
 }
 export async function openDuckdb(cfg) {
     const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
+    const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);
+    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+    rekeyChunkText(conn, store);
     return {
-        store: createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir),
+        store: store,
         cfg: resolvedCfg,
         dbPath,
         parsed,

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { clearCache } from '../sqlite/open.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately\n// (VARIANT frontmatter columns instead of untyped ones), and the store name already joins the\n// feature signature (see featureSignature), so a config's `store` key switching cleanly\n// rebuilds rather than reusing the other engine's stale cache shape.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): D1 builds DuckDB's fts index over it lazily,\n// only when a lexical query actually runs (see duckdb/lexical.ts), and reads it directly for\n// contains() substring/phrase/unspaced-script verification either way. No tokenizer resolution\n// -- this store always uses the fts extension's default (porter) stemmer, unlike sqlite's\n// configurable `content.tokenize` (not read here; D1 does not extend that knob to DuckDB).\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL\n    // (see duckdb/vectors.ts). `scale` is kept, unused, so the feature's shared reconcile-time\n    // INSERT/DELETE (src/features/embed.ts) still names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this\n  // optional peer dependency, so nothing in this store's module graph imports it as a value\n  // until a duckdb tree is actually opened (types-only imports elsewhere are erased and cost\n  // nothing either way). Installed on first use if missing, and shared with sql-functions.ts\n  // and vectors.ts (see native.ts's duckdbApi).\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which\n    // rejects VARIANT columns (\"VARIANT columns are not supported in storage versions prior to\n    // v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (see reconcile.ts), so\n    // the format floor is pinned explicitly. :memory: databases are unaffected either way.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","STORE_DIMS","SenseError","activeFeatures","FEATURES","getMeta","setMeta","clearCache","createConnection","DUCKDB_PACKAGE","duckdbApi","reconcile","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,2FAA2F;AAC3F,8FAA8F;AAC9F,wFAAwF;AACxF,qEAAqE;AACrE,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,6FAA6F;AAC7F,6FAA6F;AAC7F,+FAA+F;AAC/F,0FAA0F;AAC1F,2FAA2F;AAC3F,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWjB,eAAee,KAAM;QACzC,wFAAwF;QACxF,2FAA2F;QAC3F,yFAAyF;QACzF,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAElB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMmB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMV,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKd;AACxG;AAEA,eAAemB,QAAQL,GAAmB;IACxC,MAAMM,WAAW1B,KAAKoB,IAAIO,OAAO,EAAEzB;IACnCH,UAAU2B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS7B,KAAK0B,UAAUV;IAE9B,wFAAwF;IACxF,0FAA0F;IAC1F,2FAA2F;IAC3F,2FAA2F;IAC3F,8CAA8C;IAC9C,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe3B,YAAY,MAAM2B;QACrC,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,0FAA0F;QAC1F,2FAA2F;QAC3F,yFAAyF;QACzF,uFAAuF;QACvFA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEyB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAMhC,QAAQY,MAAM;QACpC,MAAMqB,WAAW,MAAMjC,QAAQY,MAAM;QACrC,MAAMsB,eAAexC,iBAAiBmB,KAAKd;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB5B,WAAWW;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUM,MAAMC,KAAKA,IAAIO,OAAO;QACnE,OAAO;YAAEO;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,OAAO;QAAE8B,OAAOnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;QAAGP,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","reconcile","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,8LAA8L;AAC9L,2JAA2J;AAC3J,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWjB,eAAee,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAElB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMmB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMX,QAAQW,MAAM,sBAAuB,MAAM,MAAMV,QAAQU,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMT,QAAQW,MAAM,gBAAiB,MAAM,MAAMV,QAAQU,MAAM,YAAYnB,iBAAiBoB,KAAKd;AACxG;AAEA,eAAemB,QAAQL,GAAmB;IACxC,MAAMM,WAAW3B,KAAKqB,IAAIO,OAAO,EAAE1B;IACnCH,UAAU4B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS9B,KAAK2B,UAAUV;IAE9B,gIAAgI;IAChI,qLAAqL;IACrL,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe3B,YAAY,MAAM2B;QACrC,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEyB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAM/B,QAAQW,MAAM;QACpC,MAAMqB,WAAW,MAAMhC,QAAQW,MAAM;QACrC,MAAMsB,eAAezC,iBAAiBoB,KAAKd;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB9B,WAAWa;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUM,MAAMC,KAAKA,IAAIO,OAAO;QACnE,OAAO;YAAEO;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,MAAM8B,QAAQnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;IAClF,0FAA0F;IAC1FzB,eAAeiB,MAAM+B;IACrB,OAAO;QAAEA,OAAOA;QAAO9B,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpE"}

package/dist/esm/store/duckdb/reconcile.js

@@ -5,14 +5,8 @@ import { listFiles, RESERVED_COLUMNS } from '../../scan/index.js';
 import { reparseFiles } from '../../scan/reparse.js';
 import { getColumns, quoteIdent } from '../shared.js';
 import { withTransaction } from '../transaction.js';
-// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per
-// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain
-// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from
-// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one
-// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared
-// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the
-// only DuckDB type that can hold the different JS types mapValue() produces across files for
-// the same key).
+// Mirrors sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per discovered key, ON CONFLICT upsert). `content` is
+// a plain table, not FTS-virtual, so rows are maintained here unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
 const CORE_FRONTMATTER_COLUMNS = new Set([
     'path',
     '_mtime',
@@ -31,9 +25,8 @@ function contentRow(doc) {
         doc.search.text
     ];
 }
-// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity
-// fence anyway so a runaway frontmatter generator fails with a clear message instead of an
-// unbounded ALTER TABLE loop.
+// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway
+// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.
 const MAX_FRONTMATTER_COLUMNS = 10000;
 export async function reconcile(conn, cfg, baseDir) {
     const files = listFiles(cfg, baseDir);

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per\n// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain\n// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from\n// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one\n// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared\n// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the\n// only DuckDB type that can hold the different JS types mapValue() produces across files for\n// the same key).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity\n// fence anyway so a runaway frontmatter generator fails with a clear message instead of an\n// unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,4FAA4F;AAC5F,8FAA8F;AAC9F,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iBAAiB;AACjB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,6FAA6F;AAC7F,2FAA2F;AAC3F,8BAA8B;AAC9B,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per discovered key, ON CONFLICT upsert). `content` is\n// a plain table, not FTS-virtual, so rows are maintained here unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,oIAAoI;AACpI,wOAAwO;AACxO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/duckdb/sql-functions.js

@@ -1,11 +1,8 @@
 import { segmentMatch } from '../../text/segment.js';
 import { basenameImpl, hasImpl } from '../sql-functions.js';
 import { duckdbApi } from './native.js';
-// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional
-// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a
-// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already
-// resolved the package, possibly via an on-demand install, and a second independent bare-specifier
-// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).
+// Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency (open.ts's connect()).
+// Goes through native.ts's shared duckdbApi(), not its own bare import, since a second independent lookup would hit Node's stale pre-install resolution miss.
 export async function registerFunctions(conn) {
     const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();
     conn.registerScalarFunction(DuckDBScalarFunction.create({
@@ -36,10 +33,8 @@ export async function registerFunctions(conn) {
             outputVector.flush();
         }
     }));
-    // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars
-    // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)
-    // and never calls this, but hand-written raw SQL naming it still gets the real implementation
-    // rather than a silent passthrough (PRINCIPLES: no-silent-modes).
+    // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars need (text/segment.ts); this store's own
+    // lexical index uses contains() instead and never calls this, but hand-written raw SQL naming it gets the real implementation, not a silent passthrough (PRINCIPLES: no-silent-modes).
     conn.registerScalarFunction(DuckDBScalarFunction.create({
         name: 'segment',
         parameterTypes: [

package/dist/esm/store/duckdb/sql-functions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional\n// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a\n// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already\n// resolved the package, possibly via an on-demand install, and a second independent bare-specifier\n// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars\n  // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)\n  // and never calls this, but hand-written raw SQL naming it still gets the real implementation\n  // rather than a silent passthrough (PRINCIPLES: no-silent-modes).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["segmentMatch","basenameImpl","hasImpl","duckdbApi","registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","undefined","textVec","String"],"mappings":"AACA,SAASA,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,EAAEC,OAAO,QAAQ,sBAAsB;AAC5D,SAASC,SAAS,QAAQ,cAAc;AAExC,iGAAiG;AACjG,gGAAgG;AAChG,6FAA6F;AAC7F,mGAAmG;AACnG,iGAAiG;AACjG,OAAO,eAAeC,kBAAkBC,IAAsB;IAC5D,MAAM,EAAEC,iBAAiB,EAAEC,oBAAoB,EAAEC,iBAAiB,EAAE,GAAG,MAAML;IAC7EE,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;YAAEL,kBAAkBK,QAAQ;SAAC;QACxEC,YAAYR,kBAAkBO,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMC,WAAWF,WAAWG,eAAe,CAAC;YAC5C,MAAMC,WAAWJ,WAAWG,eAAe,CAAC;YAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGpB,QAAQiB,SAASM,OAAO,CAACH,IAAID,SAASI,OAAO,CAACH;YACpHJ,aAAaQ,KAAK;QACpB;IACF;IAEFrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5Cc,aAAanB,kBAAkBK,QAAQ;QACvCC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMU,UAAUX,WAAWG,eAAe,CAAC;YAC3C,MAAMS,YAAYZ,WAAWa,WAAW,GAAG,IAAIb,WAAWG,eAAe,CAAC,KAAK;YAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGrB,aAAa2B,QAAQH,OAAO,CAACH,IAAIO,YAAYA,UAAUJ,OAAO,CAACH,KAAKS;YAC1Ib,aAAaQ,KAAK;QACpB;IACF;IAEF,0FAA0F;IAC1F,8FAA8F;IAC9F,8FAA8F;IAC9F,kEAAkE;IAClErB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5CC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gBAEgDc;YAD1F,MAAMA,UAAUf,WAAWG,eAAe,CAAC;YAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGtB,aAAaiC,QAAOD,mBAAAA,QAAQP,OAAO,CAACH,gBAAhBU,8BAAAA,mBAAsB;YAChHd,aAAaQ,KAAK;QACpB;IACF;AAEJ"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency (open.ts's connect()).\n// Goes through native.ts's shared duckdbApi(), not its own bare import, since a second independent lookup would hit Node's stale pre-install resolution miss.\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars need (text/segment.ts); this store's own\n  // lexical index uses contains() instead and never calls this, but hand-written raw SQL naming it gets the real implementation, not a silent passthrough (PRINCIPLES: no-silent-modes).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["segmentMatch","basenameImpl","hasImpl","duckdbApi","registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","undefined","textVec","String"],"mappings":"AACA,SAASA,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,EAAEC,OAAO,QAAQ,sBAAsB;AAC5D,SAASC,SAAS,QAAQ,cAAc;AAExC,mIAAmI;AACnI,8JAA8J;AAC9J,OAAO,eAAeC,kBAAkBC,IAAsB;IAC5D,MAAM,EAAEC,iBAAiB,EAAEC,oBAAoB,EAAEC,iBAAiB,EAAE,GAAG,MAAML;IAC7EE,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;YAAEL,kBAAkBK,QAAQ;SAAC;QACxEC,YAAYR,kBAAkBO,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMC,WAAWF,WAAWG,eAAe,CAAC;YAC5C,MAAMC,WAAWJ,WAAWG,eAAe,CAAC;YAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGpB,QAAQiB,SAASM,OAAO,CAACH,IAAID,SAASI,OAAO,CAACH;YACpHJ,aAAaQ,KAAK;QACpB;IACF;IAEFrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5Cc,aAAanB,kBAAkBK,QAAQ;QACvCC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMU,UAAUX,WAAWG,eAAe,CAAC;YAC3C,MAAMS,YAAYZ,WAAWa,WAAW,GAAG,IAAIb,WAAWG,eAAe,CAAC,KAAK;YAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGrB,aAAa2B,QAAQH,OAAO,CAACH,IAAIO,YAAYA,UAAUJ,OAAO,CAACH,KAAKS;YAC1Ib,aAAaQ,KAAK;QACpB;IACF;IAEF,mIAAmI;IACnI,uLAAuL;IACvLrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5CC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gBAEgDc;YAD1F,MAAMA,UAAUf,WAAWG,eAAe,CAAC;YAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGtB,aAAaiC,QAAOD,mBAAAA,QAAQP,OAAO,CAACH,gBAAhBU,8BAAAA,mBAAsB;YAChHd,aAAaQ,KAAK;QACpB;IACF;AAEJ"}

package/dist/esm/store/duckdb/store.js

@@ -7,22 +7,15 @@ import { fieldStats } from './fieldStats.js';
 import { createLexicalIndex } from './lexical.js';
 import { reconcile } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
-// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (D1: fts BM25 +
-// contains(), see lexical.ts), and vectors (D2: native FLOAT[N] + array_cosine_similarity, see
-// vectors.ts) are implemented. 'snippets' is declined: no snippet()-equivalent exists, so every
-// lexical hit relies on the caller's JS excerpt fallback (commands/search.ts) rather than a
-// bounded engine-native one. 'phrases' means quoted-phrase matching only (types.ts):
-// prefix/boolean/NEAR/column-filter FTS5 syntax is not claimed here -- lexical.ts rejects it
-// rather than answer it differently.
+// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native
+// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.
 export const CAPABILITIES = new Set([
     'lexical',
     'phrases',
     'vectors'
 ]);
-// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth
-// (see transaction.ts) is tracked against the same object everywhere. The instance is the
-// native database handle that owns the WAL: close() must close it, not just disconnect the
-// connection, or DuckDB never checkpoints and the next open reads a mismatched WAL.
+// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.
+// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.
 export function createStore(instance, duckdb, conn, cfg, baseDir) {
     const lex = createLexicalIndex(conn);
     return {
@@ -58,9 +51,8 @@ export function createStore(instance, duckdb, conn, cfg, baseDir) {
         lexical: {
             query: lex.query
         },
-        // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the
-        // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is
-        // still correct against a wider column.
+        // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --
+        // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.
         vectors: {
             pending: ()=>pendingRows(conn),
             writeVectors: (rows)=>writeVectorBatch(duckdb, conn, STORE_DIMS, rows),

package/dist/esm/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (D1: fts BM25 +\n// contains(), see lexical.ts), and vectors (D2: native FLOAT[N] + array_cosine_similarity, see\n// vectors.ts) are implemented. 'snippets' is declined: no snippet()-equivalent exists, so every\n// lexical hit relies on the caller's JS excerpt fallback (commands/search.ts) rather than a\n// bounded engine-native one. 'phrases' means quoted-phrase matching only (types.ts):\n// prefix/boolean/NEAR/column-filter FTS5 syntax is not claimed here -- lexical.ts rejects it\n// rather than answer it differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere. The instance is the\n// native database handle that owns the WAL: close() must close it, not just disconnect the\n// connection, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is\n    // still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,+FAA+F;AAC/F,gGAAgG;AAChG,4FAA4F;AAC5F,qFAAqF;AACrF,6FAA6F;AAC7F,qCAAqC;AACrC,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8FAA8F;AAC9F,0FAA0F;AAC1F,2FAA2F;AAC3F,oFAAoF;AACpF,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAMrB;YACJ,MAAMsB,SAAS,MAAMtB,UAAUS,MAAMC,KAAKC;YAC1C,uFAAuF;YACvF,4DAA4D;YAC5DC,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,sFAAsF;QACtF,0FAA0F;QAC1F,wCAAwC;QACxCC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMjB,YAAYwC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI1C,YAAY4C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMjB,YAAY+C,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8JAA8J;AAC9J,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAMrB;YACJ,MAAMsB,SAAS,MAAMtB,UAAUS,MAAMC,KAAKC;YAC1C,uFAAuF;YACvF,4DAA4D;YAC5DC,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMjB,YAAYwC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI1C,YAAY4C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMjB,YAAY+C,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}

package/dist/esm/store/duckdb/vectors.js

@@ -2,24 +2,15 @@ import { withTransaction } from '../transaction.js';
 import { asCosine, sampleEvenly } from '../vectors.js';
 import { rewriteUpdate } from './batch.js';
 import { duckdbApi } from './native.js';
-// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by
-// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with
-// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --
-// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own
-// representation and stays as it is. Every function here takes `dims` as a parameter rather
-// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real
-// constant at the call site).
-//
-// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only
-// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's
-// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever
-// open.ts already resolved, including via an on-demand install (see native.ts).
+// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and
+// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.
+// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency
+// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 const untyped = undefined;
-// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced
-// under the column's width); zero-padding leaves cosine scores unchanged since the added
-// dimensions contribute nothing to either vector's dot product or norm.
+// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);
+// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.
 function padded(values, dims) {
     const out = new Array(dims).fill(0);
     for(let d = 0; d < Math.min(values.length, dims); d++)out[d] = values[d];
@@ -30,19 +21,14 @@ function inClause(column, count) {
         length: count
     }, ()=>'?').join(', ')})`;
 }
-// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces
-// for every store, see toStore); dequantizing into a plain float array is this store's own
-// representation choice, so both stores are handed the same vectors and diverge only here.
+// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);
+// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.
 function dequantize(row, dims) {
     const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
     return padded(Array.from(q, (v)=>v * row.scale), dims);
 }
-// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's
-// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native
-// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --
-// automatic type inference on the raw values misreads an all-integer first component (a real
-// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row
-// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
+// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the
+// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
 export async function writeVectorBatch(duckdb, conn, dims, rows) {
     if (rows.length === 0) return;
     const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
@@ -64,9 +50,8 @@ export async function writeVectorBatch(duckdb, conn, dims, rows) {
         }
     });
 }
-// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks
-// the winning chunk's line range from the same row max(score) came from, per path. An empty
-// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.
+// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks the winning chunk's line range from the same row max(score) came from, per path.
+// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.
 export async function scanCandidates(duckdb, qv, dims, fetch, allowed) {
     if (allowed && allowed.size === 0) return [];
     const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
@@ -106,10 +91,8 @@ export async function scanCandidates(duckdb, qv, dims, fetch, allowed) {
         stmt.destroySync();
     }
 }
-// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed
-// into one native scan: the sampled target vectors become a small VALUES list, cross-joined
-// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost
-// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.
+// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become
+// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.
 export async function scanSimilar(duckdb, conn, dims, path, opts) {
     if (opts.allowed && opts.allowed.size === 0) return [];
     const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');