sensemaking 0.24.6 → 0.25.0

package/dist/cjs/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n  orderedMacroReady: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0', orderedMacroReady: false };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true, orderedMacroReady: false });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  state.orderedMacroReady = false;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n  if (words.length > 0) await ensureOrderedBm25(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","getMeta","stale","orderedMacroReady","set","setMeta","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","searchTokens","length","push","unspaced","split","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","foldForSearch","join","field","ORDERED_BM25_MACRO","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","matches","SenseError","ensureOrderedBm25","prepare","all","filter","row","every","some","matchesSearchPhrase","slice","path","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAiM9CA;eAAAA;;QAtIMC;eAAAA;;QAwGAC;eAAAA;;;wBAlKK;yBACwD;wBAClD;6BACD;6BAEsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtD,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAOA,kGAAkG;AAClG,kGAAkG;AAClG,IAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAAeC,SAASC,IAAgB;;YAClCC;;;;oBAAAA,QAAQJ,SAASK,GAAG,CAACF;yBACrB,CAACC,OAAD;;;;;oBAGgB;;wBAAME,IAAAA,iBAAO,EAACH,MAAMJ;;;oBAFtC,2FAA2F;oBAC3F,oFAAoF;oBACpFK,cAAUG,QAAO,AAAC,kBAA6C,UAAKC,oBAAmB;oBACvFR,SAASS,GAAG,CAACN,MAAMC;;;oBAErB;;wBAAOA;;;;IACT;;AAIO,SAAevB,iBAAiBsB,IAAgB;;;;;oBACrDH,SAASS,GAAG,CAACN,MAAM;wBAAEI,OAAO;wBAAMC,mBAAmB;oBAAM;oBAC3D;;wBAAME,IAAAA,iBAAO,EAACP,MAAMJ,oBAAoB;;;oBAAxC;;;;;;IACF;;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASY,WAAWnB,KAAa;IAC/B,IAAMoB,QAAkB,EAAE;IAC1B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAwB,EAAE;IAChC,IAAMC,aAAuB,EAAE;IAC/B,IAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,IAAMxB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACwB,IAAIC;QACtD,IAAMC,SAASD,MAAMrB,IAAI;QACzB,IAAMuB,eAAeC,IAAAA,uBAAY,EAACF;QAClC,IAAIC,aAAaE,MAAM,KAAK,GAAG;YAC7BN,cAAc;YACd,OAAO;QACT;QACAD,QAAQQ,IAAI,CAACH;YACR,kCAAA,2BAAA;;YAAL,QAAK,YAAeA,iCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;gBAA7B,IAAMxB,QAAN;gBACH,IAAIA,MAAM4B,QAAQ,EAAEV,WAAWS,IAAI,CAAC3B,MAAMV,IAAI;qBACzC;oBACHyB,MAAMY,IAAI,CAAC3B,MAAMV,IAAI;oBACrB2B,YAAYU,IAAI,CAAC3B,MAAMV,IAAI;gBAC7B;YACF;;YANK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAOL,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaM,eAAeiC,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACK,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAM9B,QAAN;oBACH,IAAIA,MAAM4B,QAAQ,EAAEV,WAAWS,IAAI,CAAC3B,MAAMV,IAAI;yBACzC;wBACHyB,MAAMY,IAAI,CAAC3B,MAAMV,IAAI;wBACrB0B,UAAUW,IAAI,CAAC3B,MAAMV,IAAI;oBAC3B;gBACF;;gBANK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAOP;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEyB,OAAAA;QAAOC,WAAAA;QAAWC,aAAAA;QAAaC,YAAAA;QAAYC,SAAAA;QAASC,aAAAA;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,SAAeW,eAAezB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMG,KAAK,EAAE;;;oBAClBH,MAAMI,iBAAiB,GAAG;oBAC1B;;wBAAML,KAAK0B,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAAC3B,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAK0B,IAAI,CAAC;;;4CAAhB;4CACA,0FAA0F;4CAC1F,gFAAgF;4CAChF;;gDAAMnB,IAAAA,iBAAO,EAACP,MAAMJ,oBAAoB;;;4CAAxC;;;;;;4BACF;;;;oBAPA;oBAQAK,MAAMG,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASwB,kBAAkBnB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIvB,MAAMW,MAAM,GAAG,GAAG;QACpB,IAAMa,YAAYxB,MAAMyB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YAC3C,kCAAA,2BAAA;;YAAL,QAAK,YAAexD,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMyD,QAAN;gBACHR,WAAWR,IAAI,CAAC,AAAC,GAAsCiB,OAApCzD,YAAY,CAACwD,MAAM,EAAC,kBAAoEA,OAApDC,iCAAkB,EAAC,oCAAuC,OAAND,OAAM;gBACjHN,YAAYV,IAAI,CAACY;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAIL,IAAIvB,UAAUU,MAAM,GAAG,GAAG;YACxB,IAAMmB,aAAa,AAAC,GAAqB,OAAnBD,iCAAkB,EAAC;YACzCR,UAAUT,IAAI,CAACkB;YACfP,WAAWX,IAAI,CAACX,UAAUwB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;QACpD;QACA,IAAIzB,YAAYS,MAAM,GAAG,GAAG;gBAK1BY;YAJA,IAAMO,cAAa,AAAC,GAAqB,OAAnBD,iCAAkB,EAAC;YACzC,IAAME,WAAW7B,YAAYuB,GAAG,CAAC,SAACO;uBAAU,AAAC,IAA6F,OAA1F7D,OAAOsD,GAAG,CAAC,SAACG;2BAAU,AAAC,wCAA6C,OAANA,OAAM;mBAASD,IAAI,CAAC,SAAQ;eAAIA,IAAI,CAAC;YACnJN,UAAUT,IAAI,CAAC,AAAC,IAAqBmB,OAAlBD,aAAW,SAAgB,OAATC,UAAS;YAC9CR,WAAWX,IAAI,CAACV,YAAYuB,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YACpDJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGrB,YAAY+B,OAAO,CAAC,SAACC;uBAAS/D,OAAOsD,GAAG,CAAC;2BAAMC,IAAAA,wBAAa,EAACQ;;;QAClF;IACF;QAEK,mCAAA,4BAAA;;;YAAA,IAAMC,MAAN;gBAOHZ;YANA,IAAMa,SAASD,IAAIE,WAAW;gBACzB,kCAAA,2BAAA;;gBAAL,QAAK,YAAelE,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;oBAAvB,IAAMyD,QAAN;oBACHR,WAAWR,IAAI,CAAC,AAAC,GAAuEgB,OAArExD,YAAY,CAACwD,MAAM,EAAC,mDAAuD,OAANA,OAAM;oBAC9FN,YAAYV,IAAI,CAACc,IAAAA,wBAAa,EAACU;gBACjC;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILf,UAAUT,IAAI,CAAC,AAAC,IAA6F,OAA1FzC,OAAOsD,GAAG,CAAC,SAACG;uBAAU,AAAC,wCAA6C,OAANA,OAAM;eAASD,IAAI,CAAC,SAAQ;YAC7GJ,CAAAA,cAAAA,YAAWX,IAAI,OAAfW,aAAgB,qBAAGpD,OAAOsD,GAAG,CAAC;uBAAMC,IAAAA,wBAAa,EAACU;;QACpD;QARA,QAAK,aAAajC,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;QAAA;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEmC,UAAUlB,WAAWO,IAAI,CAAC;QAAQY,SAASlB,UAAUM,IAAI,CAAC,YAAY;QAAQa,QAAQ,AAAC,qBAAGlB,oBAAa,qBAAGC;IAAY;AACjI;AAIO,SAAerD,aAAaqB,IAAgB,EAAEX,KAAa,EAAE6D,IAAyB,EAAEjD,KAAoB;;YAuB7FkD,OAtBdC,aAIEC,WAAWC,WAAWC,WAAWC,OACmChD,aAApEC,OAAOC,WAAWC,aAAaC,YAAYC,SAASC,aAMtBc,oBAA9BmB,UAAUC,SAASC,QACrBQ,cACAC,UACAC,KAOAR,MACAS,MACAC;;;;oBAvBAT,cAAchE,oBAAoBC;oBACxC,IAAI+D,gBAAgB,MAAM;wBACxB,MAAM,IAAIU,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEV,OAAvBA,YAAYlE,KAAK,EAAC,OAAqDG,OAAhD+D,YAAY1D,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQ0D,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACmChD,cAAAA,WAAWnB,QAA/EoB,QAAoED,YAApEC,OAAOC,YAA6DF,YAA7DE,WAAWC,cAAkDH,YAAlDG,aAAaC,aAAqCJ,YAArCI,YAAYC,UAAyBL,YAAzBK,SAASC,cAAgBN,YAAhBM;oBAC5D,IAAIA,eAAgBL,MAAMW,MAAM,KAAK,KAAKR,WAAWQ,MAAM,KAAK,GAAI;;;;yBAEhEX,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMgB,eAAezB,MAAMC;;;oBAA3B;;;yBAClBQ,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMsD,IAAAA,gCAAiB,EAAC/D,MAAMC;;;oBAA9B;;;oBAEgB2B,qBAAAA,kBAAkBnB,OAAOC,WAAWC,aAAaC,aAA/EmC,WAA8BnB,mBAA9BmB,UAAUC,UAAoBpB,mBAApBoB,SAASC,SAAWrB,mBAAXqB;oBACrBQ,eAAe5C,QAAQO,MAAM,GAAG,IAAI,mDAAmD;oBACvFsC,WAAW7C,QAAQO,MAAM,GAAG,IAAI,KAAK;oBACrCuC,MAAM,AAAC,qBACuBZ,OADHlC,QAAQO,MAAM,GAAG,IAAI,2BAA2B,IAAG,iDAC3BqC,OAArBV,UAAS,cAEzCM,OAFqDI,cAAa,4BAG5DT,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KACoBI,OADjBH,WAAU,wCACgB,OAATG;oBACrB;;wBAAM1D,KAAKgE,OAAO,CAACL;;;oBAA1BR,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKc,GAAG,OAARd,OAAAA,AAAS,qBAAGF,eAAQ,qBAAIpC,QAAQO,MAAM,GAAG;4BAAUoC;;;;oBAAjEI,OAAQ;oBACRC,UAAUhD,QAAQO,MAAM,KAAK,IAAIwC,OAAOA,KAAKM,MAAM,CAAC,SAACC;+BAAQtD,QAAQuD,KAAK,CAAC,SAACnD;mCAAWrC,OAAOyF,IAAI,CAAC,SAAChC;oCAA8B8B;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAAC9B,MAAM,cAAV8B,wBAAAA,aAAc,IAAIlD;;;;oBAC1J;;wBAAO4C,QAAQU,KAAK,CAAC,GAAGf,OAAOtB,GAAG,CAAC;gCAAGsC,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAC3D;;AAIO,SAAS/F,mBAAmBuB,IAAgB;IACjD,OAAO;QACLyE,OAAO,SAAPA,MAAcpF,OAAO6D;;;;;;;gCAAsBlD;gCAAMX;gCAAO6D;;4BAAM;;gCAAMnD,SAASC;;;;;gCAA/CrB;oCAAgC;;;;;;;QAC9D+F,WAAW,SAAXA;mBAAiBhG,iBAAiBsB;;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n  orderedMacroReady: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0', orderedMacroReady: false };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\nasync function hasFtsArtifact(conn: Connection): Promise<boolean> {\n  const stmt = await conn.prepare(`\n    SELECT 1 AS ready\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content'\n      AND function_name = 'match_bm25'\n      AND function_type = 'macro'\n    LIMIT 1\n  `);\n  return (await stmt.get()) !== undefined;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true, orderedMacroReady: false });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function rebuildFts(conn: Connection, state: FtsIndexState): Promise<void> {\n  await conn.exec('INSTALL fts; LOAD fts;');\n  if (!state.stale && (await hasFtsArtifact(conn))) return;\n  state.stale = true;\n  state.orderedMacroReady = false;\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\nexport async function prepareFts(conn: Connection): Promise<void> {\n  await rebuildFts(conn, await stateFor(conn));\n}\n\nexport async function assertFtsReady(conn: Connection): Promise<void> {\n  const state = await stateFor(conn);\n  if (state.stale) throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index is missing or stale; run `sense build` (or library build(config)) before lexical querying with build disabled');\n  try {\n    await conn.exec('LOAD fts;');\n    if (!(await hasFtsArtifact(conn))) {\n      throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index artifact is missing; run `sense build` (or library build(config)) before lexical querying with build disabled');\n    }\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('INDEX_NOT_READY', `the DuckDB lexical extension is not prepared (${(err as Error).message}); run \\`sense build\\` (or library build(config))`);\n  }\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await assertFtsReady(conn);\n  if (words.length > 0) await ensureOrderedBm25(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["assertFtsReady","createLexicalIndex","markContentStale","prepareFts","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","getMeta","stale","orderedMacroReady","set","hasFtsArtifact","stmt","prepare","undefined","setMeta","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","searchTokens","length","push","unspaced","split","tok","rebuildFts","exec","withTransaction","err","SenseError","message","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","foldForSearch","join","field","ORDERED_BM25_MACRO","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","matches","ensureOrderedBm25","all","filter","row","every","some","matchesSearchPhrase","slice","path","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAyIxCA;eAAAA;;QAuFNC;eAAAA;;QAzJMC;eAAAA;;QA8DAC;eAAAA;;QA6DAC;eAAAA;;;wBAjMK;yBACwD;wBAClD;6BACD;6BAEsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtD,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAOA,kGAAkG;AAClG,kGAAkG;AAClG,IAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAAeC,SAASC,IAAgB;;YAClCC;;;;oBAAAA,QAAQJ,SAASK,GAAG,CAACF;yBACrB,CAACC,OAAD;;;;;oBAGgB;;wBAAME,IAAAA,iBAAO,EAACH,MAAMJ;;;oBAFtC,2FAA2F;oBAC3F,oFAAoF;oBACpFK,cAAUG,QAAO,AAAC,kBAA6C,UAAKC,oBAAmB;oBACvFR,SAASS,GAAG,CAACN,MAAMC;;;oBAErB;;wBAAOA;;;;IACT;;AAEA,SAAeM,eAAeP,IAAgB;;YACtCQ;;;;oBAAO;;wBAAMR,KAAKS,OAAO,CAAC;;;oBAA1BD,OAAO;oBAQL;;wBAAMA,KAAKN,GAAG;;;oBAAtB;;wBAAQ,kBAAsBQ;;;;IAChC;;AAIO,SAAejC,iBAAiBuB,IAAgB;;;;;oBACrDH,SAASS,GAAG,CAACN,MAAM;wBAAEI,OAAO;wBAAMC,mBAAmB;oBAAM;oBAC3D;;wBAAMM,IAAAA,iBAAO,EAACX,MAAMJ,oBAAoB;;;oBAAxC;;;;;;IACF;;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASgB,WAAWvB,KAAa;IAC/B,IAAMwB,QAAkB,EAAE;IAC1B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAwB,EAAE;IAChC,IAAMC,aAAuB,EAAE;IAC/B,IAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,IAAM5B,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAAC4B,IAAIC;QACtD,IAAMC,SAASD,MAAMzB,IAAI;QACzB,IAAM2B,eAAeC,IAAAA,uBAAY,EAACF;QAClC,IAAIC,aAAaE,MAAM,KAAK,GAAG;YAC7BN,cAAc;YACd,OAAO;QACT;QACAD,QAAQQ,IAAI,CAACH;YACR,kCAAA,2BAAA;;YAAL,QAAK,YAAeA,iCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;gBAA7B,IAAM5B,QAAN;gBACH,IAAIA,MAAMgC,QAAQ,EAAEV,WAAWS,IAAI,CAAC/B,MAAMV,IAAI;qBACzC;oBACH6B,MAAMY,IAAI,CAAC/B,MAAMV,IAAI;oBACrB+B,YAAYU,IAAI,CAAC/B,MAAMV,IAAI;gBAC7B;YACF;;YANK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAOL,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaM,eAAeqC,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACK,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAMlC,QAAN;oBACH,IAAIA,MAAMgC,QAAQ,EAAEV,WAAWS,IAAI,CAAC/B,MAAMV,IAAI;yBACzC;wBACH6B,MAAMY,IAAI,CAAC/B,MAAMV,IAAI;wBACrB8B,UAAUW,IAAI,CAAC/B,MAAMV,IAAI;oBAC3B;gBACF;;gBANK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAOP;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAE6B,OAAAA;QAAOC,WAAAA;QAAWC,aAAAA;QAAaC,YAAAA;QAAYC,SAAAA;QAASC,aAAAA;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,SAAeW,WAAW7B,IAAgB,EAAEC,KAAoB;;;;;;oBAC9D;;wBAAMD,KAAK8B,IAAI,CAAC;;;oBAAhB;2BACI,CAAC7B,MAAMG,KAAK;+BAAZ;;;;oBAAiB;;wBAAMG,eAAeP;;;2BAArB;;;oBAArB,UAAkD;;;oBAClDC,MAAMG,KAAK,GAAG;oBACdH,MAAMI,iBAAiB,GAAG;oBAC1B;;wBAAM0B,IAAAA,8BAAe,EAAC/B,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAK8B,IAAI,CAAC;;;4CAAhB;4CACA,0FAA0F;4CAC1F,gFAAgF;4CAChF;;gDAAMnB,IAAAA,iBAAO,EAACX,MAAMJ,oBAAoB;;;4CAAxC;;;;;;4BACF;;;;oBAPA;oBAQAK,MAAMG,KAAK,GAAG;;;;;;IAChB;;AAEO,SAAe1B,WAAWsB,IAAgB;;;;;;;wBAC9BA;;oBAAM;;wBAAMD,SAASC;;;oBAAtC;;wBAAM6B;4BAAiB;;;;oBAAvB;;;;;;IACF;;AAEO,SAAetD,eAAeyB,IAAgB;;YAC7CC,OAOG+B;;;;oBAPK;;wBAAMjC,SAASC;;;oBAAvBC,QAAQ;oBACd,IAAIA,MAAMG,KAAK,EAAE,MAAM,IAAI6B,oBAAU,CAAC,mBAAmB;;;;;;;;;oBAEvD;;wBAAMjC,KAAK8B,IAAI,CAAC;;;oBAAhB;oBACM;;wBAAMvB,eAAeP;;;oBAA3B,IAAI,CAAE,eAA6B;wBACjC,MAAM,IAAIiC,oBAAU,CAAC,mBAAmB;oBAC1C;;;;;;oBACOD;oBACP,IAAIA,AAAG,YAAHA,KAAeC,oBAAU,GAAE,MAAMD;oBACrC,MAAM,IAAIC,oBAAU,CAAC,mBAAmB,AAAC,iDAAuE,OAAvB,AAACD,IAAcE,OAAO,EAAC;;;;;;;IAEpH;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASC,kBAAkBtB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,IAAMoB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAI1B,MAAMW,MAAM,GAAG,GAAG;QACpB,IAAMgB,YAAY3B,MAAM4B,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YAC3C,kCAAA,2BAAA;;YAAL,QAAK,YAAe/D,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMgE,QAAN;gBACHR,WAAWX,IAAI,CAAC,AAAC,GAAsCoB,OAApChE,YAAY,CAAC+D,MAAM,EAAC,kBAAoEA,OAApDC,iCAAkB,EAAC,oCAAuC,OAAND,OAAM;gBACjHN,YAAYb,IAAI,CAACe;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAIL,IAAI1B,UAAUU,MAAM,GAAG,GAAG;YACxB,IAAMsB,aAAa,AAAC,GAAqB,OAAnBD,iCAAkB,EAAC;YACzCR,UAAUZ,IAAI,CAACqB;YACfP,WAAWd,IAAI,CAACX,UAAU2B,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;QACpD;QACA,IAAI5B,YAAYS,MAAM,GAAG,GAAG;gBAK1Be;YAJA,IAAMO,cAAa,AAAC,GAAqB,OAAnBD,iCAAkB,EAAC;YACzC,IAAME,WAAWhC,YAAY0B,GAAG,CAAC,SAACO;uBAAU,AAAC,IAA6F,OAA1FpE,OAAO6D,GAAG,CAAC,SAACG;2BAAU,AAAC,wCAA6C,OAANA,OAAM;mBAASD,IAAI,CAAC,SAAQ;eAAIA,IAAI,CAAC;YACnJN,UAAUZ,IAAI,CAAC,AAAC,IAAqBsB,OAAlBD,aAAW,SAAgB,OAATC,UAAS;YAC9CR,WAAWd,IAAI,CAACV,YAAY0B,GAAG,CAACC,wBAAa,EAAEC,IAAI,CAAC;YACpDJ,CAAAA,cAAAA,YAAWd,IAAI,OAAfc,aAAgB,qBAAGxB,YAAYkC,OAAO,CAAC,SAACC;uBAAStE,OAAO6D,GAAG,CAAC;2BAAMC,IAAAA,wBAAa,EAACQ;;;QAClF;IACF;QAEK,mCAAA,4BAAA;;;YAAA,IAAMC,MAAN;gBAOHZ;YANA,IAAMa,SAASD,IAAIE,WAAW;gBACzB,kCAAA,2BAAA;;gBAAL,QAAK,YAAezE,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;oBAAvB,IAAMgE,QAAN;oBACHR,WAAWX,IAAI,CAAC,AAAC,GAAuEmB,OAArE/D,YAAY,CAAC+D,MAAM,EAAC,mDAAuD,OAANA,OAAM;oBAC9FN,YAAYb,IAAI,CAACiB,IAAAA,wBAAa,EAACU;gBACjC;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILf,UAAUZ,IAAI,CAAC,AAAC,IAA6F,OAA1F7C,OAAO6D,GAAG,CAAC,SAACG;uBAAU,AAAC,wCAA6C,OAANA,OAAM;eAASD,IAAI,CAAC,SAAQ;YAC7GJ,CAAAA,cAAAA,YAAWd,IAAI,OAAfc,aAAgB,qBAAG3D,OAAO6D,GAAG,CAAC;uBAAMC,IAAAA,wBAAa,EAACU;;QACpD;QARA,QAAK,aAAapC,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;QAAA;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEsC,UAAUlB,WAAWO,IAAI,CAAC;QAAQY,SAASlB,UAAUM,IAAI,CAAC,YAAY;QAAQa,QAAQ,AAAC,qBAAGlB,oBAAa,qBAAGC;IAAY;AACjI;AAIO,SAAe5D,aAAaqB,IAAgB,EAAEX,KAAa,EAAEoE,IAAyB,EAAExD,KAAoB;;YAuB7FO,OAtBdkD,aAIEC,WAAWC,WAAWC,WAAWC,OACmClD,aAApEC,OAAOC,WAAWC,aAAaC,YAAYC,SAASC,aAMtBiB,oBAA9BmB,UAAUC,SAASC,QACrBO,cACAC,UACAC,KAOAzD,MACA0D,MACAC;;;;oBAvBAT,cAActE,oBAAoBC;oBACxC,IAAIqE,gBAAgB,MAAM;wBACxB,MAAM,IAAIzB,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEyB,OAAvBA,YAAYxE,KAAK,EAAC,OAAqDG,OAAhDqE,YAAYhE,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQgE,YAA2CF,KAA3CE,WAAWC,YAAgCH,KAAhCG,WAAWC,YAAqBJ,KAArBI,WAAWC,QAAUL,KAAVK;oBACmClD,cAAAA,WAAWvB,QAA/EwB,QAAoED,YAApEC,OAAOC,YAA6DF,YAA7DE,WAAWC,cAAkDH,YAAlDG,aAAaC,aAAqCJ,YAArCI,YAAYC,UAAyBL,YAAzBK,SAASC,cAAgBN,YAAhBM;oBAC5D,IAAIA,eAAgBL,MAAMW,MAAM,KAAK,KAAKR,WAAWQ,MAAM,KAAK,GAAI;;;;yBAEhEX,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMtC,eAAeyB;;;oBAArB;;;yBAClBa,CAAAA,MAAMW,MAAM,GAAG,CAAA,GAAfX;;;;oBAAkB;;wBAAMuD,IAAAA,gCAAiB,EAACpE,MAAMC;;;oBAA9B;;;oBAEgBkC,qBAAAA,kBAAkBtB,OAAOC,WAAWC,aAAaC,aAA/EsC,WAA8BnB,mBAA9BmB,UAAUC,UAAoBpB,mBAApBoB,SAASC,SAAWrB,mBAAXqB;oBACrBO,eAAe9C,QAAQO,MAAM,GAAG,IAAI,mDAAmD;oBACvFwC,WAAW/C,QAAQO,MAAM,GAAG,IAAI,KAAK;oBACrCyC,MAAM,AAAC,qBACuBX,OADHrC,QAAQO,MAAM,GAAG,IAAI,2BAA2B,IAAG,iDAC3BuC,OAArBT,UAAS,cAEzCK,OAFqDI,cAAa,4BAG5DR,OADNI,WAAU,gBAEVC,OADML,SAAQ,UACDM,OAAbD,WAAU,KACoBI,OADjBH,WAAU,wCACgB,OAATG;oBACrB;;wBAAMhE,KAAKS,OAAO,CAACwD;;;oBAA1BzD,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAK6D,GAAG,OAAR7D,OAAAA,AAAS,qBAAGgD,eAAQ,qBAAIvC,QAAQO,MAAM,GAAG;4BAAUsC;;;;oBAAjEI,OAAQ;oBACRC,UAAUlD,QAAQO,MAAM,KAAK,IAAI0C,OAAOA,KAAKI,MAAM,CAAC,SAACC;+BAAQtD,QAAQuD,KAAK,CAAC,SAACnD;mCAAWzC,OAAO6F,IAAI,CAAC,SAAC7B;oCAA8B2B;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAAC3B,MAAM,cAAV2B,wBAAAA,aAAc,IAAIlD;;;;oBAC1J;;wBAAO8C,QAAQQ,KAAK,CAAC,GAAGb,OAAOrB,GAAG,CAAC;gCAAGmC,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAC3D;;AAIO,SAASpG,mBAAmBwB,IAAgB;IACjD,OAAO;QACL6E,OAAO,SAAPA,MAAcxF,OAAOoE;;;;;;;gCAAsBzD;gCAAMX;gCAAOoE;;4BAAM;;gCAAM1D,SAASC;;;;;gCAA/CrB;oCAAgC;;;;;;;QAC9DmG,WAAW,SAAXA;mBAAiBrG,iBAAiBuB;;IACpC;AACF"}

package/dist/cjs/store/duckdb/open.d.cts

@@ -1,13 +1,13 @@
 import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { OpenResult } from '../open.js';
+import type { InternalOpenOptions, OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "4";
+export declare const SCHEMA_VERSION = "5";
 export type { OpenResult };
 interface DuckdbHandle {
     instance: DuckDBInstance;
     duckdb: DuckDBConnection;
 }
 export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
-export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult>;

package/dist/cjs/store/duckdb/open.d.ts

@@ -1,13 +1,13 @@
 import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { OpenResult } from '../open.js';
+import type { InternalOpenOptions, OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "4";
+export declare const SCHEMA_VERSION = "5";
 export type { OpenResult };
 interface DuckdbHandle {
     instance: DuckDBInstance;
     duckdb: DuckDBConnection;
 }
 export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
-export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult>;

package/dist/cjs/store/duckdb/open.js

@@ -29,10 +29,12 @@ var _indexts1 = require("../../features/index.js");
 var _opents = require("../open.js");
 var _sharedts = require("../shared.js");
 var _connectionts = require("./connection.js");
+var _lexicalts = require("./lexical.js");
 var _nativets = require("./native.js");
 var _reconcilets = require("./reconcile.js");
 var _sqlfunctionsts = require("./sql-functions.js");
 var _storets = require("./store.js");
+var _vectorsts = require("./vectors.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -165,10 +167,11 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.duckdb';
-var SCHEMA_VERSION = '4';
-// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
+var SCHEMA_VERSION = '5';
+// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,
+// while contains() verification reads the table directly.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
-function ensureSchema(_handle, conn, cfg) {
+function ensureSchema(handle, conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
         return _ts_generator(this, function(_state) {
@@ -188,75 +191,87 @@ function ensureSchema(_handle, conn, cfg) {
                     _state.sent();
                     return [
                         4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))')
+                        conn.exec('CREATE TABLE IF NOT EXISTS indexed_sources ("path" TEXT PRIMARY KEY, text TEXT NOT NULL)')
                     ];
                 case 3:
                     _state.sent();
                     return [
                         4,
-                        conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)')
+                        conn.exec('CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))')
                     ];
                 case 4:
                     _state.sent();
-                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                    _state.label = 5;
+                    return [
+                        4,
+                        conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)')
+                    ];
                 case 5:
+                    _state.sent();
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 6;
+                case 6:
                     _state.trys.push([
-                        5,
-                        12,
-                        13,
-                        14
+                        6,
+                        14,
+                        15,
+                        16
                     ]);
                     _iterator = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
-                    _state.label = 6;
-                case 6:
+                    _state.label = 7;
+                case 7:
                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
                         3,
-                        11
+                        13
                     ];
                     feature = _step.value;
                     if (!(feature.name === 'embed')) return [
                         3,
-                        8
+                        10
                     ];
                     return [
                         4,
                         conn.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT['.concat(_typests.STORE_DIMS, '], PRIMARY KEY ("path", chunk))'))
                     ];
-                case 7:
+                case 8:
+                    _state.sent();
+                    return [
+                        4,
+                        (0, _vectorsts.createVectorWriteStage)(handle.duckdb, _typests.STORE_DIMS)
+                    ];
+                case 9:
                     _state.sent();
                     return [
                         3,
-                        10
+                        12
                     ];
-                case 8:
+                case 10:
                     return [
                         4,
                         feature.schema(conn)
                     ];
-                case 9:
+                case 11:
                     _state.sent();
-                    _state.label = 10;
-                case 10:
+                    _state.label = 12;
+                case 12:
                     _iteratorNormalCompletion = true;
                     return [
                         3,
-                        6
+                        7
                     ];
-                case 11:
+                case 13:
                     return [
                         3,
-                        14
+                        16
                     ];
-                case 12:
+                case 14:
                     err = _state.sent();
                     _didIteratorError = true;
                     _iteratorError = err;
                     return [
                         3,
-                        14
+                        16
                     ];
-                case 13:
+                case 15:
                     try {
                         if (!_iteratorNormalCompletion && _iterator.return != null) {
                             _iterator.return();
@@ -269,41 +284,41 @@ function ensureSchema(_handle, conn, cfg) {
                     return [
                         7
                     ];
-                case 14:
+                case 16:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'schema_version')
                     ];
-                case 15:
+                case 17:
                     if (!(_state.sent() === null)) return [
                         3,
-                        17
+                        19
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'schema_version', SCHEMA_VERSION)
                     ];
-                case 16:
+                case 18:
                     _state.sent();
-                    _state.label = 17;
-                case 17:
+                    _state.label = 19;
+                case 19:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'features')
                     ];
-                case 18:
+                case 20:
                     if (!(_state.sent() === null)) return [
                         3,
-                        20
+                        22
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES))
                     ];
-                case 19:
+                case 21:
                     _state.sent();
-                    _state.label = 20;
-                case 20:
+                    _state.label = 22;
+                case 22:
                     return [
                         2
                     ];
@@ -323,7 +338,7 @@ function close(handle) {
         });
     })();
 }
-function connect(dbPath, _cfg) {
+function connect(dbPath, _cfg, options) {
     return _async_to_generator(function() {
         var DuckDBInstance, err, duckdb, instance, err1, conn, err2;
         return _ts_generator(this, function(_state) {
@@ -358,7 +373,9 @@ function connect(dbPath, _cfg) {
                     ]);
                     return [
                         4,
-                        DuckDBInstance.create(dbPath, {
+                        DuckDBInstance.create(dbPath, options.observational ? {
+                            access_mode: 'READ_ONLY'
+                        } : {
                             storage_compatibility_version: 'v1.5.0'
                         })
                     ];
@@ -438,16 +455,18 @@ var duckdbOpenDialect = {
         return /Could not set lock on file|being used by another process/.test(err.message);
     },
     ensureSchema: ensureSchema,
+    prepareLexical: _lexicalts.prepareFts,
+    assertLexicalReady: _lexicalts.assertFtsReady,
     createStore: function createStore(handle, conn) {
         return (0, _storets.createStore)(handle.instance, handle.duckdb, conn);
     }
 };
-function openDuckdb(cfg) {
+function openDuckdb(cfg, options) {
     return _async_to_generator(function() {
         return _ts_generator(this, function(_state) {
             return [
                 2,
-                (0, _opents.openWithDialect)(cfg, duckdbOpenDialect)
+                (0, _opents.openWithDialect)(cfg, duckdbOpenDialect, options)
             ];
         });
     })();

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","createVectorWriteStage","duckdb","schema","getMeta","setMeta","featureSignature","FEATURES","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","options","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","observational","access_mode","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","prepareLexical","prepareFts","assertLexicalReady","assertFtsReady","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAkBaA;eAAAA;;QAGAC;eAAAA;;QA2EAC;eAAAA;;QAgBSC;eAAAA;;;uBA9GW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;yBACU;wBACD;2BACZ;8BACI;uBACN;yBACW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEhC,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,SAAeG,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;;YAMxE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBALX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;wBAAMC,IAAAA,iCAAsB,EAACR,OAAOS,MAAM,EAAEF,mBAAU;;;oBAAtD;oBACA;;;;;oBAEF;;wBAAMJ,QAAQO,MAAM,CAACT;;;oBAArB;;;oBARG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAUA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMf,MAAoB;;;YACvCA,OAAOS,MAAM,CAACO,cAAc;YAC5BhB,OAAOiB,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB,EAAEC,OAA0D;;YAGjHC,gBAGKC,KAKLf,QACAQ,UAMKO,MAQDvB,MAECuB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACT,QAAQE,QAAQQ,aAAa,GAAG;4BAAEC,aAAa;wBAAY,IAAI;4BAAEC,+BAA+B;wBAAS;;;oBAFhJ,4HAA4H;oBAC5H,4KAA4K;oBAC5Kf,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BV,SAAS;;;;;;oBACFe;oBACP,wFAAwF;oBACxFP,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIQ,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXN,QAAO,MAA2B,OAAvB,AAACI,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMK,IAAAA,iCAAiB,EAACxB;;;oBAAxB;oBACMR,OAAOiC,IAAAA,8BAAgB,EAACzB;oBAC9B;;wBAAO;4BAAET,QAAQ;gCAAEiB,UAAAA;gCAAUR,QAAAA;4BAAO;4BAAGR,MAAAA;wBAAK;;;oBACrCuB;oBACP;;wBAAMT,MAAM;4BAAEE,UAAAA;4BAAUR,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMe;;;;;;;IAEV;;AAGO,IAAM3B,oBAA+C;IAC1DsC,UAAUxC;IACVyC,eAAexC;IACfyC,kBAAkBC,0BAAa;IAC/BnB,SAAAA;IACAJ,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFwB,UAAU,SAAVA,SAAWf;eAAQ,2DAA2DgB,IAAI,CAAChB,IAAII,OAAO;;IAC9F7B,cAAAA;IACA0C,gBAAgBC,qBAAU;IAC1BC,oBAAoBC,yBAAc;IAClCC,aAAa,SAAbA,YAAc7C,QAAQC;eAAS4C,IAAAA,oBAAW,EAAC7C,OAAOiB,QAAQ,EAAEjB,OAAOS,MAAM,EAAER;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB,EAAEoB,OAA6B;;;YACjF;;gBAAOwB,IAAAA,uBAAe,EAAC5C,KAAKL,mBAAmByB;;;IACjD"}

package/dist/cjs/store/duckdb/reconcile.js

@@ -189,8 +189,8 @@ function reconcileContent(conn, touched, docs, _delta) {
                     ];
                 case 3:
                     _state.sent();
-                    // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
-                    // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.
+                    // Build prepares the FTS index after reconcile. This persisted mark commits with the content
+                    // change, so an interrupted build cannot expose stale lexical results.
                     return [
                         4,
                         (0, _lexicalts.markContentStale)(conn)

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","appendRows","markContentStale","addColumns","names","exec","name","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAiDaA;;;eAAAA;;;wBAjDc;wBAGY;yBAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAE9F,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQb,GAAG,CAAC,SAACkB;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMC,IAAAA,oBAAU,EAACP,MAAM,WAAWd,iBAAiBC,oBAAoBe,KAAKd,GAAG,CAACG;;;oBAAhF;oBACA,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMiB,IAAAA,2BAAgB,EAACR;;;oBAAvB;;;;;;IACF;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMtB,GAAG,CAAC,SAACwB;mCAAS,AAAC,sCAAsD,OAAjBvB,IAAAA,oBAAU,EAACuB,OAAM;2BAAWtB,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C4B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjB,yBAAyB;YACnC,MAAM,IAAIkB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElB,OAAtDiB,OAAM,kDAAwE,OAAxBjB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // Build prepares the FTS index after reconcile. This persisted mark commits with the content\n  // change, so an interrupted build cannot expose stale lexical results.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","appendRows","markContentStale","addColumns","names","exec","name","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAiDaA;;;eAAAA;;;wBAjDc;wBAGY;yBAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAE9F,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQb,GAAG,CAAC,SAACkB;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMC,IAAAA,oBAAU,EAACP,MAAM,WAAWd,iBAAiBC,oBAAoBe,KAAKd,GAAG,CAACG;;;oBAAhF;oBACA,6FAA6F;oBAC7F,uEAAuE;oBACvE;;wBAAMiB,IAAAA,2BAAgB,EAACR;;;oBAAvB;;;;;;IACF;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMtB,GAAG,CAAC,SAACwB;mCAAS,AAAC,sCAAsD,OAAjBvB,IAAAA,oBAAU,EAACuB,OAAM;2BAAWtB,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C4B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjB,yBAAyB;YACnC,MAAM,IAAIkB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElB,OAAtDiB,OAAM,kDAAwE,OAAxBjB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}

package/dist/cjs/store/duckdb/vectors.d.cts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/cjs/store/duckdb/vectors.d.ts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {