sensemaking 0.27.1 → 0.28.1

package/dist/esm/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { type Config, featureSignature, type ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.ts';\nimport { getMeta } from '../store/shared.ts';\nimport { embeddingSignatureCompatible } from '../store/signature.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { identifyChunkText, takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { type EmbedCallOptions, STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nfunction staleEmbeddingWork(reason: string): SenseError {\n  return new SenseError('INDEX_NOT_READY', `${reason}; run \\`sense build\\` (or library build(config)) again to prepare the current index generation`);\n}\n\n// expectedSignature is captured by open() before provider work. Direct internal callers omit it\n// and capture the durable value here, before selecting chunks or constructing a provider.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options: EmbedCallOptions = {}): Promise<void> {\n  options.signal?.throwIfAborted();\n  const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;\n  options.signal?.throwIfAborted();\n  if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {\n    throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');\n  }\n\n  const pendingStmt = await store.prepare('SELECT \"path\", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  const dirty = ((await pendingStmt.all()) as Array<{ path: string; chunk: number; content_identity: string }>).filter((row) => allowed?.has(row.path) ?? true);\n  options.signal?.throwIfAborted();\n  if (dirty.length === 0) return;\n\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const submittedSignature = featureSignature(cfg, FEATURES);\n  if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {\n    throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');\n  }\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, typeof dirty>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string; identity: string }> = [];\n  for (const [path, pending] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && pending.every((row) => texts[row.chunk]?.identity === row.content_identity)) {\n      for (const row of pending) jobs.push({ path, chunk: row.chunk, ...texts[row.chunk] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    options.signal?.throwIfAborted();\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const row of pending) {\n      const chunk = chunks[row.chunk];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      const identified = identifyChunkText(chunk.text);\n      if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);\n      jobs.push({ path, chunk: row.chunk, ...identified });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  options.signal?.throwIfAborted();\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n  options.signal?.throwIfAborted();\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');\n    const vectors = await provider.embedDocuments(\n      batch.map((j) => j.text),\n      options\n    );\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');\n    const writes = batch.map((job, j) => {\n      const { v, scale } = toStore(vectors[j], storeDims, true);\n      const q = new Int8Array(storeDims);\n      for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n      return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n    });\n    options.signal?.throwIfAborted();\n    await store.transaction(async () => {\n      // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws\n      // inside this transaction and rolls every row in the provider batch back together.\n      options.signal?.throwIfAborted();\n      await store.vectors.writeVectors(writes);\n      options.signal?.throwIfAborted();\n      if ((await getMeta(store, FEATURE_SIGNATURE_META_KEY)) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');\n      const currentStmt = await store.prepare(`SELECT \"path\", chunk, content_identity FROM embeddings WHERE ${batch.map(() => '(\"path\" = ? AND chunk = ?)').join(' OR ')}`);\n      const current = (await currentStmt.all(...batch.flatMap((job) => [job.path, job.chunk]))) as Array<{ path: string; chunk: number; content_identity: string }>;\n      const identities = new Map(current.map((row) => [`${row.path}\\0${Number(row.chunk)}`, row.content_identity]));\n      if (batch.some((job) => identities.get(`${job.path}\\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');\n      options.signal?.throwIfAborted();\n    });\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\nexport interface PreparedSemanticQuery {\n  readonly vector: Float32Array;\n  readonly storeDims: number;\n}\n\n// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as\n// lexical syntax before the query is embedded, matching the prior semantic query behavior.\nexport async function prepareSemanticQuery(cfg: Config, terms: string, options: EmbedCallOptions = {}): Promise<PreparedSemanticQuery> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  options.signal?.throwIfAborted();\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const queryVector = await provider.embedQuery(text, options);\n  options.signal?.throwIfAborted();\n  const { v: qv } = toStore(queryVector, storeDims, false);\n  return { vector: qv, storeDims };\n}\n\n// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot\n// and the prepared vector, so this performs no provider work and only delegates to the store.\nexport function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["featureSignature","SenseError","embed","FEATURES","progress","parseSource","FEATURE_SIGNATURE_META_KEY","getMeta","embeddingSignatureCompatible","identifyChunkText","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","staleEmbeddingWork","reason","embedPending","store","cfg","allowed","expectedSignature","options","signal","throwIfAborted","expectedDurableSignature","undefined","pendingStmt","prepare","dirty","all","filter","row","has","path","length","provider","submittedSignature","storeDims","min","byPath","Map","list","get","push","set","textByPath","sourceStmt","jobs","pending","texts","every","chunk","identity","content_identity","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","text","presets","doc","extracted","identified","map","j","report","i","batchCap","batch","slice","vectors","embedDocuments","writes","job","q","Int8Array","round","vector","from","buffer","transaction","writeVectors","currentStmt","join","current","flatMap","identities","Number","some","tick","finish","ensureDocumentEmbeddings","_cfg","prepareSemanticQuery","terms","rootDir","baseDir","match","t","includes","queryVector","embedQuery","qv","semanticCandidates","prepared","fetch","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AAAA,SAAsBA,gBAAgB,QAA6B,qBAAqB;AACxF,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAC/C,SAASC,0BAA0B,QAAQ,wBAAwB;AACnE,SAASC,OAAO,QAAQ,qBAAqB;AAC7C,SAASC,4BAA4B,QAAQ,wBAAwB;AAErE,SAASC,iBAAiB,EAAEC,aAAa,QAAQ,eAAe;AAChE,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAAgCC,UAAU,QAAQ,aAAa;AAE/D,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,SAASE,mBAAmBC,MAAc;IACxC,OAAO,IAAI3B,WAAW,mBAAmB,GAAG2B,OAAO,8FAA8F,CAAC;AACpJ;AAEA,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,eAAeC,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B,EAAEC,iBAAiC,EAAEC,UAA4B,CAAC,CAAC;QA4BzIxB;QA3BnBwB,iBAEAA,kBAOAA,kBAIAA,kBAuCA,sFAAsF;IACtF,2EAA2E;IAC3EA,kBAMAA;KA5DAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMC,2BAA2BJ,sBAAsBK,YAAY,MAAM/B,QAAQuB,OAAOxB,8BAA8B2B;KACtHC,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAI,CAAC5B,6BAA6B6B,0BAA0BrC,iBAAiB+B,KAAK5B,YAAY;QAC5F,MAAMwB,mBAAmB;IAC3B;IAEA,MAAMY,cAAc,MAAMT,MAAMU,OAAO,CAAC;IACxC,MAAMC,QAAQ,AAAE,CAAA,MAAMF,YAAYG,GAAG,EAAC,EAAwEC,MAAM,CAAC,CAACC;;uBAAQZ,oBAAAA,8BAAAA,QAASa,GAAG,CAACD,IAAIE,IAAI,wCAAK;;KACxJZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAIK,MAAMM,MAAM,KAAK,GAAG;IAExB,MAAMC,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMa,qBAAqBjD,iBAAiB+B,KAAK5B;IACjD,IAAI,CAACK,6BAA6B6B,0BAA0BY,qBAAqB;QAC/E,MAAMtB,mBAAmB;IAC3B;IACA,MAAMuB,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IAEpD,MAAMoC,SAAS,IAAIC;IACnB,KAAK,MAAMT,OAAOH,MAAO;YACVW;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACX,IAAIE,IAAI,eAAnBM,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACZ;QACVQ,OAAOK,GAAG,CAACb,IAAIE,IAAI,EAAEQ;IACvB;IAEA,MAAMI,cAAahD,iBAAAA,cAAcoB,oBAAdpB,4BAAAA,iBAAwB,IAAI2C;IAC/C,MAAMM,aAAa,MAAM7B,MAAMU,OAAO,CAAC;IAEvC,MAAMoB,OAA+E,EAAE;IACvF,KAAK,MAAM,CAACd,MAAMe,QAAQ,IAAIT,OAAQ;YASpClB;QARA,MAAM4B,QAAQJ,WAAWH,GAAG,CAACT;QAC7B,IAAIgB,SAASD,QAAQE,KAAK,CAAC,CAACnB;gBAAQkB;mBAAAA,EAAAA,mBAAAA,KAAK,CAAClB,IAAIoB,KAAK,CAAC,cAAhBF,uCAAAA,iBAAkBG,QAAQ,MAAKrB,IAAIsB,gBAAgB;YAAG;YACxF,KAAK,MAAMtB,OAAOiB,QAASD,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGF,KAAK,CAAClB,IAAIoB,KAAK,CAAC;YAAC;YACnF;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAMG,SAAU,MAAMR,WAAWJ,GAAG,CAACT;SACrCZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAI,CAAC+B,QAAQ,MAAM,IAAIlE,WAAW,mBAAmB,CAAC,mCAAmC,EAAE6C,KAAK,kGAAkG,CAAC;QACnM,MAAMsB,SAAS/D,YAAY;YAAEgE,SAASvB;YAAMwB,SAASxB;YAAMyB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOS,IAAI;YAAGC,SAAS,EAAE;YAAE3E,OAAO;QAAK,GAAGiE,OAAOS,IAAI,EAAE;YAAC1E;SAAM,EAAE6B,KAAK+C,GAAG,CAACC,SAAS,CAAC7E,KAAK;QACnM,KAAK,MAAM0C,OAAOiB,QAAS;YACzB,MAAMG,QAAQI,MAAM,CAACxB,IAAIoB,KAAK,CAAC;YAC/B,IAAI,CAACA,OAAO,MAAM,IAAI/D,WAAW,mBAAmB,CAAC,4BAA4B,EAAE6C,KAAK,2GAA2G,CAAC;YACpM,MAAMkC,aAAavE,kBAAkBuD,MAAMY,IAAI;YAC/C,IAAII,WAAWf,QAAQ,KAAKrB,IAAIsB,gBAAgB,EAAE,MAAMvC,mBAAmB,CAAC,cAAc,EAAEmB,KAAK,CAAC,EAAEF,IAAIoB,KAAK,CAAC,yCAAyC,CAAC;YACxJJ,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGgB,UAAU;YAAC;QACpD;IACF;KAIA9C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMzB,iBACJmB,OACAkB,UACAY,KAAKqB,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI;KAExB1C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAE9B,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAM+C,SAAS/E,SAAS,oBAAoBwD,KAAKb,MAAM;IACvD,IAAK,IAAIqC,IAAI,GAAGA,IAAIxB,KAAKb,MAAM,EAAEqC,KAAKpC,SAASqC,QAAQ,CAAE;YAEvDnD,kBAMAA,kBAQAA;QAfA,MAAMoD,QAAQ1B,KAAK2B,KAAK,CAACH,GAAGA,IAAIpC,SAASqC,QAAQ;SACjDnD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM6D,UAAU,MAAMxC,SAASyC,cAAc,CAC3CH,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI,GACvB1C;SAEFA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM+D,SAASJ,MAAML,GAAG,CAAC,CAACU,KAAKT;YAC7B,MAAM,EAAEhE,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQ0E,OAAO,CAACN,EAAE,EAAEhC,WAAW;YACpD,MAAM0C,IAAI,IAAIC,UAAU3C;YACxB,IAAK,IAAI7B,IAAI,GAAGA,IAAI6B,WAAW7B,IAAKuE,CAAC,CAACvE,EAAE,GAAGC,KAAKwE,KAAK,CAAC5E,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEsB,MAAM6C,IAAI7C,IAAI;gBAAEkB,OAAO2B,IAAI3B,KAAK;gBAAExC;gBAAOuE,QAAQrB,OAAOsB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;SACA/D,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,MAAMN,MAAMoE,WAAW,CAAC;gBACtB,wFAAwF;YACxF,mFAAmF;YACnFhE,iBAEAA,kBAMAA;aARAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;YAC9B,MAAMN,MAAM0D,OAAO,CAACW,YAAY,CAACT;aACjCxD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;YAC9B,IAAI,AAAC,MAAM7B,QAAQuB,OAAOxB,gCAAiC+B,0BAA0B,MAAMV,mBAAmB;YAC9G,MAAMyE,cAAc,MAAMtE,MAAMU,OAAO,CAAC,CAAC,6DAA6D,EAAE8C,MAAML,GAAG,CAAC,IAAM,8BAA8BoB,IAAI,CAAC,SAAS;YACpK,MAAMC,UAAW,MAAMF,YAAY1D,GAAG,IAAI4C,MAAMiB,OAAO,CAAC,CAACZ,MAAQ;oBAACA,IAAI7C,IAAI;oBAAE6C,IAAI3B,KAAK;iBAAC;YACtF,MAAMwC,aAAa,IAAInD,IAAIiD,QAAQrB,GAAG,CAAC,CAACrC,MAAQ;oBAAC,GAAGA,IAAIE,IAAI,CAAC,EAAE,EAAE2D,OAAO7D,IAAIoB,KAAK,GAAG;oBAAEpB,IAAIsB,gBAAgB;iBAAC;YAC3G,IAAIoB,MAAMoB,IAAI,CAAC,CAACf,MAAQa,WAAWjD,GAAG,CAAC,GAAGoC,IAAI7C,IAAI,CAAC,EAAE,EAAE6C,IAAI3B,KAAK,EAAE,MAAM2B,IAAI1B,QAAQ,GAAG,MAAMtC,mBAAmB;aAChHO,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAChC;QACA+C,OAAOwB,IAAI,CAACrF,KAAK6B,GAAG,CAACiC,IAAIpC,SAASqC,QAAQ,EAAEzB,KAAKb,MAAM;IACzD;IACAoC,OAAOyB,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB/E,KAAY,EAAEgF,IAAY,EAAE9E,OAA4B;IACrG,MAAM6B,UAAU,AAAC,CAAA,MAAM/B,MAAM0D,OAAO,CAAC3B,OAAO,EAAC,EAAGlB,MAAM,CAAC,CAACC,MAAQZ,QAAQa,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIe,QAAQd,MAAM,GAAG,GAAG;QACtB,MAAM,IAAI9C,WAAW,mBAAmB,GAAG4D,QAAQd,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAOA,2FAA2F;AAC3F,2FAA2F;AAC3F,OAAO,eAAegE,qBAAqBhF,GAAW,EAAEiF,KAAa,EAAE9E,UAA4B,CAAC,CAAC;QACnF,cAEF8E;QACd9E,iBAEAA,kBAGAA;IARA,MAAM+E,WAAU,eAAA,AAAClF,IAAgCkF,OAAO,cAAxC,0BAAA,eAA4C,AAAClF,IAAgCmF,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIhH,WAAW,eAAe;IAClD,MAAM2E,OAAO,EAACoC,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAErE,MAAM,CAAC,CAACyE,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIf,IAAI,CAAC;KAClHnE,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMY,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMc,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IACpD,MAAMsG,cAAc,MAAMtE,SAASuE,UAAU,CAAC3C,MAAM1C;KACpDA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAM,EAAElB,GAAGsG,EAAE,EAAE,GAAG1G,QAAQwG,aAAapE,WAAW;IAClD,OAAO;QAAE6C,QAAQyB;QAAItE;IAAU;AACjC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,OAAO,SAASuE,mBAAmB3F,KAAY,EAAE4F,QAA+B,EAAEC,KAAa,EAAE3F,OAAqB;IACpH,OAAOF,MAAM0D,OAAO,CAACoC,UAAU,CAACF,SAAS3B,MAAM,EAAE2B,SAASxE,SAAS,EAAEyE,OAAO3F;AAC9E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAe6F,aAAa/F,KAAY,EAAEgB,IAAY;IAC3D,OAAOhB,MAAM0D,OAAO,CAACsC,SAAS,CAAChF;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiF,aAAajG,KAAY,EAAEgF,IAAY,EAAEhE,IAAY,EAAEkF,IAAgE;IAC3I,OAAOlG,MAAM0D,OAAO,CAACyC,OAAO,CAACnF,MAAMkF;AACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { type Config, featureSignature, type ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.ts';\nimport { getMeta } from '../store/shared.ts';\nimport { embeddingSignatureCompatible } from '../store/signature.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { identifyChunkText, takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { type EmbedCallOptions, STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.ts, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nfunction staleEmbeddingWork(reason: string): SenseError {\n  return new SenseError('INDEX_NOT_READY', `${reason}; run \\`sense build\\` (or library build(config)) again to prepare the current index generation`);\n}\n\n// expectedSignature is captured by open() before provider work. Direct internal callers omit it\n// and capture the durable value here, before selecting chunks or constructing a provider.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options: EmbedCallOptions = {}): Promise<void> {\n  options.signal?.throwIfAborted();\n  const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;\n  options.signal?.throwIfAborted();\n  if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {\n    throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');\n  }\n\n  const pendingStmt = await store.prepare('SELECT \"path\", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  const dirty = ((await pendingStmt.all()) as Array<{ path: string; chunk: number; content_identity: string }>).filter((row) => allowed?.has(row.path) ?? true);\n  options.signal?.throwIfAborted();\n  if (dirty.length === 0) return;\n\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const submittedSignature = featureSignature(cfg, FEATURES);\n  if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {\n    throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');\n  }\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, typeof dirty>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string; identity: string }> = [];\n  for (const [path, pending] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && pending.every((row) => texts[row.chunk]?.identity === row.content_identity)) {\n      for (const row of pending) jobs.push({ path, chunk: row.chunk, ...texts[row.chunk] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    options.signal?.throwIfAborted();\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const row of pending) {\n      const chunk = chunks[row.chunk];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      const identified = identifyChunkText(chunk.text);\n      if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);\n      jobs.push({ path, chunk: row.chunk, ...identified });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  options.signal?.throwIfAborted();\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n  options.signal?.throwIfAborted();\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');\n    const vectors = await provider.embedDocuments(\n      batch.map((j) => j.text),\n      options\n    );\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');\n    const writes = batch.map((job, j) => {\n      const { v, scale } = toStore(vectors[j], storeDims, true);\n      const q = new Int8Array(storeDims);\n      for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n      return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n    });\n    options.signal?.throwIfAborted();\n    await store.transaction(async () => {\n      // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws\n      // inside this transaction and rolls every row in the provider batch back together.\n      options.signal?.throwIfAborted();\n      await store.vectors.writeVectors(writes);\n      options.signal?.throwIfAborted();\n      if ((await getMeta(store, FEATURE_SIGNATURE_META_KEY)) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');\n      const currentStmt = await store.prepare(`SELECT \"path\", chunk, content_identity FROM embeddings WHERE ${batch.map(() => '(\"path\" = ? AND chunk = ?)').join(' OR ')}`);\n      const current = (await currentStmt.all(...batch.flatMap((job) => [job.path, job.chunk]))) as Array<{ path: string; chunk: number; content_identity: string }>;\n      const identities = new Map(current.map((row) => [`${row.path}\\0${Number(row.chunk)}`, row.content_identity]));\n      if (batch.some((job) => identities.get(`${job.path}\\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');\n      options.signal?.throwIfAborted();\n    });\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\nexport interface PreparedSemanticQuery {\n  readonly vector: Float32Array;\n  readonly storeDims: number;\n}\n\n// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as\n// lexical syntax before the query is embedded, matching the prior semantic query behavior.\nexport async function prepareSemanticQuery(cfg: Config, terms: string, options: EmbedCallOptions = {}): Promise<PreparedSemanticQuery> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  options.signal?.throwIfAborted();\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const queryVector = await provider.embedQuery(text, options);\n  options.signal?.throwIfAborted();\n  const { v: qv } = toStore(queryVector, storeDims, false);\n  return { vector: qv, storeDims };\n}\n\n// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot\n// and the prepared vector, so this performs no provider work and only delegates to the store.\nexport function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["featureSignature","SenseError","embed","FEATURES","progress","parseSource","FEATURE_SIGNATURE_META_KEY","getMeta","embeddingSignatureCompatible","identifyChunkText","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","staleEmbeddingWork","reason","embedPending","store","cfg","allowed","expectedSignature","options","signal","throwIfAborted","expectedDurableSignature","undefined","pendingStmt","prepare","dirty","all","filter","row","has","path","length","provider","submittedSignature","storeDims","min","byPath","Map","list","get","push","set","textByPath","sourceStmt","jobs","pending","texts","every","chunk","identity","content_identity","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","text","presets","doc","extracted","identified","map","j","report","i","batchCap","batch","slice","vectors","embedDocuments","writes","job","q","Int8Array","round","vector","from","buffer","transaction","writeVectors","currentStmt","join","current","flatMap","identities","Number","some","tick","finish","ensureDocumentEmbeddings","_cfg","prepareSemanticQuery","terms","rootDir","baseDir","match","t","includes","queryVector","embedQuery","qv","semanticCandidates","prepared","fetch","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AAAA,SAAsBA,gBAAgB,QAA6B,qBAAqB;AACxF,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAC/C,SAASC,0BAA0B,QAAQ,wBAAwB;AACnE,SAASC,OAAO,QAAQ,qBAAqB;AAC7C,SAASC,4BAA4B,QAAQ,wBAAwB;AAErE,SAASC,iBAAiB,EAAEC,aAAa,QAAQ,eAAe;AAChE,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAAgCC,UAAU,QAAQ,aAAa;AAE/D,yFAAyF;AACzF,oEAAoE;AACpE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,SAASE,mBAAmBC,MAAc;IACxC,OAAO,IAAI3B,WAAW,mBAAmB,GAAG2B,OAAO,8FAA8F,CAAC;AACpJ;AAEA,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,eAAeC,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B,EAAEC,iBAAiC,EAAEC,UAA4B,CAAC,CAAC;QA4BzIxB;QA3BnBwB,iBAEAA,kBAOAA,kBAIAA,kBAuCA,sFAAsF;IACtF,2EAA2E;IAC3EA,kBAMAA;KA5DAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMC,2BAA2BJ,sBAAsBK,YAAY,MAAM/B,QAAQuB,OAAOxB,8BAA8B2B;KACtHC,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAI,CAAC5B,6BAA6B6B,0BAA0BrC,iBAAiB+B,KAAK5B,YAAY;QAC5F,MAAMwB,mBAAmB;IAC3B;IAEA,MAAMY,cAAc,MAAMT,MAAMU,OAAO,CAAC;IACxC,MAAMC,QAAQ,AAAE,CAAA,MAAMF,YAAYG,GAAG,EAAC,EAAwEC,MAAM,CAAC,CAACC;;uBAAQZ,oBAAAA,8BAAAA,QAASa,GAAG,CAACD,IAAIE,IAAI,wCAAK;;KACxJZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAIK,MAAMM,MAAM,KAAK,GAAG;IAExB,MAAMC,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMa,qBAAqBjD,iBAAiB+B,KAAK5B;IACjD,IAAI,CAACK,6BAA6B6B,0BAA0BY,qBAAqB;QAC/E,MAAMtB,mBAAmB;IAC3B;IACA,MAAMuB,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IAEpD,MAAMoC,SAAS,IAAIC;IACnB,KAAK,MAAMT,OAAOH,MAAO;YACVW;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACX,IAAIE,IAAI,eAAnBM,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACZ;QACVQ,OAAOK,GAAG,CAACb,IAAIE,IAAI,EAAEQ;IACvB;IAEA,MAAMI,cAAahD,iBAAAA,cAAcoB,oBAAdpB,4BAAAA,iBAAwB,IAAI2C;IAC/C,MAAMM,aAAa,MAAM7B,MAAMU,OAAO,CAAC;IAEvC,MAAMoB,OAA+E,EAAE;IACvF,KAAK,MAAM,CAACd,MAAMe,QAAQ,IAAIT,OAAQ;YASpClB;QARA,MAAM4B,QAAQJ,WAAWH,GAAG,CAACT;QAC7B,IAAIgB,SAASD,QAAQE,KAAK,CAAC,CAACnB;gBAAQkB;mBAAAA,EAAAA,mBAAAA,KAAK,CAAClB,IAAIoB,KAAK,CAAC,cAAhBF,uCAAAA,iBAAkBG,QAAQ,MAAKrB,IAAIsB,gBAAgB;YAAG;YACxF,KAAK,MAAMtB,OAAOiB,QAASD,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGF,KAAK,CAAClB,IAAIoB,KAAK,CAAC;YAAC;YACnF;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAMG,SAAU,MAAMR,WAAWJ,GAAG,CAACT;SACrCZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAI,CAAC+B,QAAQ,MAAM,IAAIlE,WAAW,mBAAmB,CAAC,mCAAmC,EAAE6C,KAAK,kGAAkG,CAAC;QACnM,MAAMsB,SAAS/D,YAAY;YAAEgE,SAASvB;YAAMwB,SAASxB;YAAMyB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOS,IAAI;YAAGC,SAAS,EAAE;YAAE3E,OAAO;QAAK,GAAGiE,OAAOS,IAAI,EAAE;YAAC1E;SAAM,EAAE6B,KAAK+C,GAAG,CAACC,SAAS,CAAC7E,KAAK;QACnM,KAAK,MAAM0C,OAAOiB,QAAS;YACzB,MAAMG,QAAQI,MAAM,CAACxB,IAAIoB,KAAK,CAAC;YAC/B,IAAI,CAACA,OAAO,MAAM,IAAI/D,WAAW,mBAAmB,CAAC,4BAA4B,EAAE6C,KAAK,2GAA2G,CAAC;YACpM,MAAMkC,aAAavE,kBAAkBuD,MAAMY,IAAI;YAC/C,IAAII,WAAWf,QAAQ,KAAKrB,IAAIsB,gBAAgB,EAAE,MAAMvC,mBAAmB,CAAC,cAAc,EAAEmB,KAAK,CAAC,EAAEF,IAAIoB,KAAK,CAAC,yCAAyC,CAAC;YACxJJ,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGgB,UAAU;YAAC;QACpD;IACF;KAIA9C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMzB,iBACJmB,OACAkB,UACAY,KAAKqB,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI;KAExB1C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAE9B,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAM+C,SAAS/E,SAAS,oBAAoBwD,KAAKb,MAAM;IACvD,IAAK,IAAIqC,IAAI,GAAGA,IAAIxB,KAAKb,MAAM,EAAEqC,KAAKpC,SAASqC,QAAQ,CAAE;YAEvDnD,kBAMAA,kBAQAA;QAfA,MAAMoD,QAAQ1B,KAAK2B,KAAK,CAACH,GAAGA,IAAIpC,SAASqC,QAAQ;SACjDnD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM6D,UAAU,MAAMxC,SAASyC,cAAc,CAC3CH,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI,GACvB1C;SAEFA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM+D,SAASJ,MAAML,GAAG,CAAC,CAACU,KAAKT;YAC7B,MAAM,EAAEhE,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQ0E,OAAO,CAACN,EAAE,EAAEhC,WAAW;YACpD,MAAM0C,IAAI,IAAIC,UAAU3C;YACxB,IAAK,IAAI7B,IAAI,GAAGA,IAAI6B,WAAW7B,IAAKuE,CAAC,CAACvE,EAAE,GAAGC,KAAKwE,KAAK,CAAC5E,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEsB,MAAM6C,IAAI7C,IAAI;gBAAEkB,OAAO2B,IAAI3B,KAAK;gBAAExC;gBAAOuE,QAAQrB,OAAOsB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;SACA/D,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,MAAMN,MAAMoE,WAAW,CAAC;gBACtB,wFAAwF;YACxF,mFAAmF;YACnFhE,iBAEAA,kBAMAA;aARAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;YAC9B,MAAMN,MAAM0D,OAAO,CAACW,YAAY,CAACT;aACjCxD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;YAC9B,IAAI,AAAC,MAAM7B,QAAQuB,OAAOxB,gCAAiC+B,0BAA0B,MAAMV,mBAAmB;YAC9G,MAAMyE,cAAc,MAAMtE,MAAMU,OAAO,CAAC,CAAC,6DAA6D,EAAE8C,MAAML,GAAG,CAAC,IAAM,8BAA8BoB,IAAI,CAAC,SAAS;YACpK,MAAMC,UAAW,MAAMF,YAAY1D,GAAG,IAAI4C,MAAMiB,OAAO,CAAC,CAACZ,MAAQ;oBAACA,IAAI7C,IAAI;oBAAE6C,IAAI3B,KAAK;iBAAC;YACtF,MAAMwC,aAAa,IAAInD,IAAIiD,QAAQrB,GAAG,CAAC,CAACrC,MAAQ;oBAAC,GAAGA,IAAIE,IAAI,CAAC,EAAE,EAAE2D,OAAO7D,IAAIoB,KAAK,GAAG;oBAAEpB,IAAIsB,gBAAgB;iBAAC;YAC3G,IAAIoB,MAAMoB,IAAI,CAAC,CAACf,MAAQa,WAAWjD,GAAG,CAAC,GAAGoC,IAAI7C,IAAI,CAAC,EAAE,EAAE6C,IAAI3B,KAAK,EAAE,MAAM2B,IAAI1B,QAAQ,GAAG,MAAMtC,mBAAmB;aAChHO,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAChC;QACA+C,OAAOwB,IAAI,CAACrF,KAAK6B,GAAG,CAACiC,IAAIpC,SAASqC,QAAQ,EAAEzB,KAAKb,MAAM;IACzD;IACAoC,OAAOyB,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB/E,KAAY,EAAEgF,IAAY,EAAE9E,OAA4B;IACrG,MAAM6B,UAAU,AAAC,CAAA,MAAM/B,MAAM0D,OAAO,CAAC3B,OAAO,EAAC,EAAGlB,MAAM,CAAC,CAACC,MAAQZ,QAAQa,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIe,QAAQd,MAAM,GAAG,GAAG;QACtB,MAAM,IAAI9C,WAAW,mBAAmB,GAAG4D,QAAQd,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAOA,2FAA2F;AAC3F,2FAA2F;AAC3F,OAAO,eAAegE,qBAAqBhF,GAAW,EAAEiF,KAAa,EAAE9E,UAA4B,CAAC,CAAC;QACnF,cAEF8E;QACd9E,iBAEAA,kBAGAA;IARA,MAAM+E,WAAU,eAAA,AAAClF,IAAgCkF,OAAO,cAAxC,0BAAA,eAA4C,AAAClF,IAAgCmF,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIhH,WAAW,eAAe;IAClD,MAAM2E,OAAO,EAACoC,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAErE,MAAM,CAAC,CAACyE,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIf,IAAI,CAAC;KAClHnE,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMY,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMc,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IACpD,MAAMsG,cAAc,MAAMtE,SAASuE,UAAU,CAAC3C,MAAM1C;KACpDA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAM,EAAElB,GAAGsG,EAAE,EAAE,GAAG1G,QAAQwG,aAAapE,WAAW;IAClD,OAAO;QAAE6C,QAAQyB;QAAItE;IAAU;AACjC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,OAAO,SAASuE,mBAAmB3F,KAAY,EAAE4F,QAA+B,EAAEC,KAAa,EAAE3F,OAAqB;IACpH,OAAOF,MAAM0D,OAAO,CAACoC,UAAU,CAACF,SAAS3B,MAAM,EAAE2B,SAASxE,SAAS,EAAEyE,OAAO3F;AAC9E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAe6F,aAAa/F,KAAY,EAAEgB,IAAY;IAC3D,OAAOhB,MAAM0D,OAAO,CAACsC,SAAS,CAAChF;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiF,aAAajG,KAAY,EAAEgF,IAAY,EAAEhE,IAAY,EAAEkF,IAAgE;IAC3I,OAAOlG,MAAM0D,OAAO,CAACyC,OAAO,CAACnF,MAAMkF;AACrC"}

package/dist/esm/store/duckdb/engine.d.ts

@@ -0,0 +1,11 @@
+import { type EnginePatch } from '../engine.js';
+export declare const FTS_STEMMER = "porter";
+export interface DuckdbFtsBuilders {
+    createIndex(): string;
+    queryTerms(): string;
+    termScore(field: string): string;
+    conjunctiveGate(): string;
+}
+export declare const DUCKDB_FTS_BUILDERS: DuckdbFtsBuilders;
+export declare const DUCKDB_PATCHES: readonly EnginePatch<DuckdbFtsBuilders>[];
+export declare function duckdbFts(): Promise<DuckdbFtsBuilders>;

package/dist/esm/store/duckdb/engine.js

@@ -0,0 +1,18 @@
+import { selectBuilders } from '../engine.js';
+import { duckdbNative } from './native.js';
+export const FTS_STEMMER = 'porter';
+export const DUCKDB_FTS_BUILDERS = {
+    createIndex: ()=>`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stemmer='${FTS_STEMMER}', stopwords='none', overwrite=1)`,
+    queryTerms: ()=>`
+    WITH tokens AS (SELECT unnest(fts_main_content.tokenize(?)) AS raw_term)
+    SELECT min(raw_term) AS term FROM tokens
+    WHERE raw_term <> '' GROUP BY stem(raw_term, '${FTS_STEMMER}') ORDER BY term
+  `,
+    termScore: (field)=>`SELECT fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false) AS score`,
+    conjunctiveGate: ()=>`fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`
+};
+// Empty: DuckDB 1.4.5, 1.5.0 and 1.5.6 produce identical macros, tokens and scores (probed 2026-10-11).
+export const DUCKDB_PATCHES = [];
+export async function duckdbFts() {
+    return selectBuilders(DUCKDB_FTS_BUILDERS, DUCKDB_PATCHES, (await duckdbNative()).version);
+}

package/dist/esm/store/duckdb/engine.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/engine.ts"],"sourcesContent":["import { type EnginePatch, selectBuilders } from '../engine.ts';\nimport { duckdbNative } from './native.ts';\n\nexport const FTS_STEMMER = 'porter';\n\n// Every statement this store sends to the fts extension; a version that behaves differently gets a DUCKDB_PATCHES entry.\n// match_bm25 tokenizes its query with the generated fts_main_content.tokenize macro, so queryTerms uses that same macro.\nexport interface DuckdbFtsBuilders {\n  // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords; the extension's default list would\n  // drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n  createIndex(): string;\n  // One bound parameter, the folded query; returns one original token per stem so native scoring stems once.\n  queryTerms(): string;\n  // One bound parameter, a single query term, scored within one field.\n  termScore(field: string): string;\n  // One bound parameter, the space-joined words that must all occur.\n  conjunctiveGate(): string;\n}\n\nexport const DUCKDB_FTS_BUILDERS: DuckdbFtsBuilders = {\n  createIndex: () => `PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stemmer='${FTS_STEMMER}', stopwords='none', overwrite=1)`,\n  queryTerms: () => `\n    WITH tokens AS (SELECT unnest(fts_main_content.tokenize(?)) AS raw_term)\n    SELECT min(raw_term) AS term FROM tokens\n    WHERE raw_term <> '' GROUP BY stem(raw_term, '${FTS_STEMMER}') ORDER BY term\n  `,\n  termScore: (field) => `SELECT fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false) AS score`,\n  conjunctiveGate: () => `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`,\n};\n\n// Empty: DuckDB 1.4.5, 1.5.0 and 1.5.6 produce identical macros, tokens and scores (probed 2026-10-11).\nexport const DUCKDB_PATCHES: readonly EnginePatch<DuckdbFtsBuilders>[] = [];\n\nexport async function duckdbFts(): Promise<DuckdbFtsBuilders> {\n  return selectBuilders(DUCKDB_FTS_BUILDERS, DUCKDB_PATCHES, (await duckdbNative()).version);\n}\n"],"names":["selectBuilders","duckdbNative","FTS_STEMMER","DUCKDB_FTS_BUILDERS","createIndex","queryTerms","termScore","field","conjunctiveGate","DUCKDB_PATCHES","duckdbFts","version"],"mappings":"AAAA,SAA2BA,cAAc,QAAQ,eAAe;AAChE,SAASC,YAAY,QAAQ,cAAc;AAE3C,OAAO,MAAMC,cAAc,SAAS;AAgBpC,OAAO,MAAMC,sBAAyC;IACpDC,aAAa,IAAM,CAAC,gFAAgF,EAAEF,YAAY,iCAAiC,CAAC;IACpJG,YAAY,IAAM,CAAC;;;kDAG6B,EAAEH,YAAY;EAC9D,CAAC;IACDI,WAAW,CAACC,QAAU,CAAC,iEAAiE,EAAEA,MAAM,iCAAiC,CAAC;IAClIC,iBAAiB,IAAM,CAAC,+EAA+E,CAAC;AAC1G,EAAE;AAEF,wGAAwG;AACxG,OAAO,MAAMC,iBAA4D,EAAE,CAAC;AAE5E,OAAO,eAAeC;IACpB,OAAOV,eAAeG,qBAAqBM,gBAAgB,AAAC,CAAA,MAAMR,cAAa,EAAGU,OAAO;AAC3F"}

package/dist/esm/store/duckdb/lexical.js

@@ -4,14 +4,15 @@ import { SenseError } from '../../errors.js';
 import { foldForSearch, matchesSearchPhrase, searchTokens } from '../../text/segment.js';
 import { getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
-import { FTS_STEMMER, nativeQueryTerms, orderedBm25Sql } from './ordered-bm25.js';
+import { duckdbFts } from './engine.js';
+import { nativeQueryTerms, orderedBm25Sql } from './ordered-bm25.js';
 const FIELDS = [
     'title',
     'summary',
     'text'
 ];
 // Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no
-// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.
+// field_weights argument (verified 1.4.5 through 1.5.6), so each field is scored separately and combined here.
 const FIELD_WEIGHT = {
     title: 10,
     summary: 5,
@@ -145,9 +146,7 @@ async function rebuildFts(conn, state) {
     if (!state.stale && await hasFtsArtifact(conn)) return;
     state.stale = true;
     await withTransaction(conn, async ()=>{
-        // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's
-        // default 571-word list would otherwise drop common words (e.g. "and") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.
-        await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stemmer='${FTS_STEMMER}', stopwords='none', overwrite=1)`);
+        await conn.exec((await duckdbFts()).createIndex());
         // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant
         // rebuild next time), never a cleared mark serving a half-built index as fresh.
         await setMeta(conn, FTS_STALE_META_KEY, '0');
@@ -172,23 +171,23 @@ export async function assertFtsReady(conn) {
 }
 // Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
 // score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
-function buildScoreAndGate(words, nativeTerms, bareWords, phraseWords, substrings) {
+function buildScoreAndGate(fts, words, nativeTerms, bareWords, phraseWords, substrings) {
     const scoreParts = [];
     const gateParts = [];
     const scoreParams = [];
     const gateParams = [];
     if (words.length > 0) {
         for (const field of FIELDS){
-            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${orderedBm25Sql(field, nativeTerms.length)}, 0)`);
+            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${orderedBm25Sql(fts, field, nativeTerms.length)}, 0)`);
             scoreParams.push(...nativeTerms);
         }
         if (bareWords.length > 0) {
-            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const nativeGate = fts.conjunctiveGate();
             gateParts.push(nativeGate);
             gateParams.push(bareWords.map(foldForSearch).join(' '));
         }
         if (phraseWords.length > 0) {
-            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const nativeGate = fts.conjunctiveGate();
             const fallback = phraseWords.map((_word)=>`(${FIELDS.map((field)=>`contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');
             gateParts.push(`(${nativeGate} OR (${fallback}))`);
             gateParams.push(phraseWords.map(foldForSearch).join(' '));
@@ -224,8 +223,9 @@ export async function queryLexical(conn, terms, opts) {
     const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);
     if (emptyPhrase || words.length === 0 && substrings.length === 0) return [];
     if (words.length > 0) await assertFtsReady(conn);
-    const nativeTerms = words.length > 0 ? await nativeQueryTerms(conn, words.map(foldForSearch).join(' ')) : [];
-    const { scoreSql, gateSql, params } = buildScoreAndGate(words, nativeTerms, bareWords, phraseWords, substrings);
+    const fts = await duckdbFts();
+    const nativeTerms = words.length > 0 ? await nativeQueryTerms(conn, fts, words.map(foldForSearch).join(' ')) : [];
+    const { scoreSql, gateSql, params } = buildScoreAndGate(fts, words, nativeTerms, bareWords, phraseWords, substrings);
     const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
     const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';
     const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { FTS_STEMMER, nativeQueryTerms, orderedBm25Sql } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0' };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\nasync function hasFtsArtifact(conn: Connection): Promise<boolean> {\n  const stmt = await conn.prepare(`\n    SELECT 1 AS ready\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content'\n      AND function_name = 'match_bm25'\n      AND function_type = 'macro'\n    LIMIT 1\n  `);\n  return (await stmt.get()) !== undefined;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function rebuildFts(conn: Connection, state: FtsIndexState): Promise<void> {\n  await conn.exec('INSTALL fts; LOAD fts;');\n  if (!state.stale && (await hasFtsArtifact(conn))) return;\n  state.stale = true;\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stemmer='${FTS_STEMMER}', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\nexport async function prepareFts(conn: Connection): Promise<void> {\n  await rebuildFts(conn, await stateFor(conn));\n}\n\nexport async function assertFtsReady(conn: Connection): Promise<void> {\n  const state = await stateFor(conn);\n  if (state.stale) throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index is missing or stale; run `sense build` (or library build(config)) before lexical querying with build disabled');\n  try {\n    await conn.exec('LOAD fts;');\n    if (!(await hasFtsArtifact(conn))) {\n      throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index artifact is missing; run `sense build` (or library build(config)) before lexical querying with build disabled');\n    }\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('INDEX_NOT_READY', `the DuckDB lexical extension is not prepared (${(err as Error).message}); run \\`sense build\\` (or library build(config))`);\n  }\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], nativeTerms: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${orderedBm25Sql(field, nativeTerms.length)}, 0)`);\n      scoreParams.push(...nativeTerms);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await assertFtsReady(conn);\n  const nativeTerms = words.length > 0 ? await nativeQueryTerms(conn, words.map(foldForSearch).join(' ')) : [];\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, nativeTerms, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","FTS_STEMMER","nativeQueryTerms","orderedBm25Sql","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","set","hasFtsArtifact","stmt","prepare","undefined","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","rebuildFts","exec","prepareFts","assertFtsReady","err","message","buildScoreAndGate","nativeTerms","scoreParts","gateParts","scoreParams","gateParams","field","nativeGate","map","join","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,WAAW,EAAEC,gBAAgB,EAAEC,cAAc,QAAQ,oBAAoB;AAElF,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM7B,QAAQ0B,MAAMJ,wBAAyB;QAAI;QACnEC,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,eAAeI,eAAeL,IAAgB;IAC5C,MAAMM,OAAO,MAAMN,KAAKO,OAAO,CAAC,CAAC;;;;;;;EAOjC,CAAC;IACD,OAAO,AAAC,MAAMD,KAAKJ,GAAG,OAAQM;AAChC;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeC,iBAAiBT,IAAgB;IACrDH,SAASO,GAAG,CAACJ,MAAM;QAAEG,OAAO;IAAK;IACjC,MAAM5B,QAAQyB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASc,WAAWrB,KAAa;IAC/B,MAAMsB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAM1B,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAAC0B,IAAIC;QACtD,MAAMC,SAASD,MAAMvB,IAAI;QACzB,MAAMyB,eAAe/C,aAAa8C;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAM1B,SAAS0B,aAAc;YAChC,IAAI1B,MAAM6B,QAAQ,EAAET,WAAWQ,IAAI,CAAC5B,MAAMV,IAAI;iBACzC;gBACH2B,MAAMW,IAAI,CAAC5B,MAAMV,IAAI;gBACrB6B,YAAYS,IAAI,CAAC5B,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMwC,OAAOlC,eAAemC,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAM3B,SAASrB,aAAamD,KAAM;YACrC,IAAI9B,MAAM6B,QAAQ,EAAET,WAAWQ,IAAI,CAAC5B,MAAMV,IAAI;iBACzC;gBACH2B,MAAMW,IAAI,CAAC5B,MAAMV,IAAI;gBACrB4B,UAAUU,IAAI,CAAC5B,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAE2B;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,WAAW1B,IAAgB,EAAEC,KAAoB;IAC9D,MAAMD,KAAK2B,IAAI,CAAC;IAChB,IAAI,CAAC1B,MAAME,KAAK,IAAK,MAAME,eAAeL,OAAQ;IAClDC,MAAME,KAAK,GAAG;IACd,MAAM3B,gBAAgBwB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAK2B,IAAI,CAAC,CAAC,gFAAgF,EAAElD,YAAY,iCAAiC,CAAC;QACjJ,0FAA0F;QAC1F,gFAAgF;QAChF,MAAMF,QAAQyB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,OAAO,eAAeyB,WAAW5B,IAAgB;IAC/C,MAAM0B,WAAW1B,MAAM,MAAMD,SAASC;AACxC;AAEA,OAAO,eAAe6B,eAAe7B,IAAgB;IACnD,MAAMC,QAAQ,MAAMF,SAASC;IAC7B,IAAIC,MAAME,KAAK,EAAE,MAAM,IAAIjC,WAAW,mBAAmB;IACzD,IAAI;QACF,MAAM8B,KAAK2B,IAAI,CAAC;QAChB,IAAI,CAAE,MAAMtB,eAAeL,OAAQ;YACjC,MAAM,IAAI9B,WAAW,mBAAmB;QAC1C;IACF,EAAE,OAAO4D,KAAK;QACZ,IAAIA,eAAe5D,YAAY,MAAM4D;QACrC,MAAM,IAAI5D,WAAW,mBAAmB,CAAC,8CAA8C,EAAE,AAAC4D,IAAcC,OAAO,CAAC,iDAAiD,CAAC;IACpK;AACF;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASC,kBAAkBrB,KAAe,EAAEsB,WAAqB,EAAErB,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IACjI,MAAMoB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAI1B,MAAMU,MAAM,GAAG,GAAG;QACpB,KAAK,MAAMiB,SAAS1D,OAAQ;YAC1BsD,WAAWZ,IAAI,CAAC,GAAGzC,YAAY,CAACyD,MAAM,CAAC,cAAc,EAAE3D,eAAe2D,OAAOL,YAAYZ,MAAM,EAAE,IAAI,CAAC;YACtGe,YAAYd,IAAI,IAAIW;QACtB;QACA,IAAIrB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMkB,aAAa,CAAC,+EAA+E,CAAC;YACpGJ,UAAUb,IAAI,CAACiB;YACfF,WAAWf,IAAI,CAACV,UAAU4B,GAAG,CAACrE,eAAesE,IAAI,CAAC;QACpD;QACA,IAAI5B,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMkB,aAAa,CAAC,+EAA+E,CAAC;YACpG,MAAMG,WAAW7B,YAAY2B,GAAG,CAAC,CAACG,QAAU,CAAC,CAAC,EAAE/D,OAAO4D,GAAG,CAAC,CAACF,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAEG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJN,UAAUb,IAAI,CAAC,CAAC,CAAC,EAAEiB,WAAW,KAAK,EAAEG,SAAS,EAAE,CAAC;YACjDL,WAAWf,IAAI,CAACT,YAAY2B,GAAG,CAACrE,eAAesE,IAAI,CAAC;YACpDJ,WAAWf,IAAI,IAAIT,YAAY+B,OAAO,CAAC,CAACC,OAASjE,OAAO4D,GAAG,CAAC,IAAMrE,cAAc0E;QAClF;IACF;IAEA,KAAK,MAAMC,OAAOhC,WAAY;QAC5B,MAAMiC,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMV,SAAS1D,OAAQ;YAC1BsD,WAAWZ,IAAI,CAAC,GAAGzC,YAAY,CAACyD,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHF,YAAYd,IAAI,CAACnD,cAAc4E;QACjC;QACAZ,UAAUb,IAAI,CAAC,CAAC,CAAC,EAAE1C,OAAO4D,GAAG,CAAC,CAACF,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAEG,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GJ,WAAWf,IAAI,IAAI1C,OAAO4D,GAAG,CAAC,IAAMrE,cAAc4E;IACpD;IAEA,OAAO;QAAEE,UAAUf,WAAWO,IAAI,CAAC;QAAQS,SAASf,UAAUM,IAAI,CAAC,YAAY;QAAQU,QAAQ;eAAIf;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAee,aAAapD,IAAgB,EAAEX,KAAa,EAAEgE,IAAyB;IAC3F,MAAMC,cAAclE,oBAAoBC;IACxC,IAAIiE,gBAAgB,MAAM;QACxB,MAAM,IAAIpF,WAAW,4BAA4B,CAAC,yCAAyC,EAAEoF,YAAYpE,KAAK,CAAC,GAAG,EAAEoE,YAAY5D,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAE4D,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE1C,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWrB;IACvF,IAAI2B,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMQ,eAAe7B;IAC3C,MAAMiC,cAActB,MAAMU,MAAM,GAAG,IAAI,MAAM3C,iBAAiBsB,MAAMW,MAAM6B,GAAG,CAACrE,eAAesE,IAAI,CAAC,QAAQ,EAAE;IAE5G,MAAM,EAAEQ,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGnB,kBAAkBrB,OAAOsB,aAAarB,WAAWC,aAAaC;IACpG,MAAM6C,eAAe5C,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMuC,WAAW7C,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAMwC,MAAM,CAAC,kBAAkB,EAAE9C,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAE4B,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAMtD,OAAO,MAAMN,KAAKO,OAAO,CAACsD;IAChC,MAAMC,OAAQ,MAAMxD,KAAKyD,GAAG,IAAIZ,WAAYpC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACqC;KAAM;IAC7E,MAAMM,UAAUjD,QAAQM,MAAM,KAAK,IAAIyC,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQnD,QAAQoD,KAAK,CAAC,CAAChD,SAAWvC,OAAOwF,IAAI,CAAC,CAAC9B;oBAA8B4B;uBAApB9F,qBAAoB8F,aAAAA,GAAG,CAAC5B,MAAM,cAAV4B,wBAAAA,aAAc,IAAI/C;;IAC1J,OAAO6C,QAAQK,KAAK,CAAC,GAAGX,OAAOlB,GAAG,CAAC,CAAC,EAAE8B,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmBvE,IAAgB;IACjD,OAAO;QACLwE,OAAO,OAAOnF,OAAOgE,OAASD,aAAapD,MAAMX,OAAOgE;QACxDoB,WAAW,IAAMhE,iBAAiBT;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { type DuckdbFtsBuilders, duckdbFts } from './engine.ts';\nimport { nativeQueryTerms, orderedBm25Sql } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.4.5 through 1.5.6), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0' };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\nasync function hasFtsArtifact(conn: Connection): Promise<boolean> {\n  const stmt = await conn.prepare(`\n    SELECT 1 AS ready\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content'\n      AND function_name = 'match_bm25'\n      AND function_type = 'macro'\n    LIMIT 1\n  `);\n  return (await stmt.get()) !== undefined;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function rebuildFts(conn: Connection, state: FtsIndexState): Promise<void> {\n  await conn.exec('INSTALL fts; LOAD fts;');\n  if (!state.stale && (await hasFtsArtifact(conn))) return;\n  state.stale = true;\n  await withTransaction(conn, async () => {\n    await conn.exec((await duckdbFts()).createIndex());\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\nexport async function prepareFts(conn: Connection): Promise<void> {\n  await rebuildFts(conn, await stateFor(conn));\n}\n\nexport async function assertFtsReady(conn: Connection): Promise<void> {\n  const state = await stateFor(conn);\n  if (state.stale) throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index is missing or stale; run `sense build` (or library build(config)) before lexical querying with build disabled');\n  try {\n    await conn.exec('LOAD fts;');\n    if (!(await hasFtsArtifact(conn))) {\n      throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index artifact is missing; run `sense build` (or library build(config)) before lexical querying with build disabled');\n    }\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('INDEX_NOT_READY', `the DuckDB lexical extension is not prepared (${(err as Error).message}); run \\`sense build\\` (or library build(config))`);\n  }\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(fts: DuckdbFtsBuilders, words: string[], nativeTerms: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${orderedBm25Sql(fts, field, nativeTerms.length)}, 0)`);\n      scoreParams.push(...nativeTerms);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = fts.conjunctiveGate();\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = fts.conjunctiveGate();\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await assertFtsReady(conn);\n  const fts = await duckdbFts();\n  const nativeTerms = words.length > 0 ? await nativeQueryTerms(conn, fts, words.map(foldForSearch).join(' ')) : [];\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(fts, words, nativeTerms, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","duckdbFts","nativeQueryTerms","orderedBm25Sql","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","set","hasFtsArtifact","stmt","prepare","undefined","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","rebuildFts","exec","createIndex","prepareFts","assertFtsReady","err","message","buildScoreAndGate","fts","nativeTerms","scoreParts","gateParts","scoreParams","gateParams","field","nativeGate","conjunctiveGate","map","join","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAAiCC,SAAS,QAAQ,cAAc;AAChE,SAASC,gBAAgB,EAAEC,cAAc,QAAQ,oBAAoB;AAErE,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,+GAA+G;AAC/G,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM7B,QAAQ0B,MAAMJ,wBAAyB;QAAI;QACnEC,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,eAAeI,eAAeL,IAAgB;IAC5C,MAAMM,OAAO,MAAMN,KAAKO,OAAO,CAAC,CAAC;;;;;;;EAOjC,CAAC;IACD,OAAO,AAAC,MAAMD,KAAKJ,GAAG,OAAQM;AAChC;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeC,iBAAiBT,IAAgB;IACrDH,SAASO,GAAG,CAACJ,MAAM;QAAEG,OAAO;IAAK;IACjC,MAAM5B,QAAQyB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASc,WAAWrB,KAAa;IAC/B,MAAMsB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAM1B,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAAC0B,IAAIC;QACtD,MAAMC,SAASD,MAAMvB,IAAI;QACzB,MAAMyB,eAAe/C,aAAa8C;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAM1B,SAAS0B,aAAc;YAChC,IAAI1B,MAAM6B,QAAQ,EAAET,WAAWQ,IAAI,CAAC5B,MAAMV,IAAI;iBACzC;gBACH2B,MAAMW,IAAI,CAAC5B,MAAMV,IAAI;gBACrB6B,YAAYS,IAAI,CAAC5B,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMwC,OAAOlC,eAAemC,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAM3B,SAASrB,aAAamD,KAAM;YACrC,IAAI9B,MAAM6B,QAAQ,EAAET,WAAWQ,IAAI,CAAC5B,MAAMV,IAAI;iBACzC;gBACH2B,MAAMW,IAAI,CAAC5B,MAAMV,IAAI;gBACrB4B,UAAUU,IAAI,CAAC5B,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAE2B;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,WAAW1B,IAAgB,EAAEC,KAAoB;IAC9D,MAAMD,KAAK2B,IAAI,CAAC;IAChB,IAAI,CAAC1B,MAAME,KAAK,IAAK,MAAME,eAAeL,OAAQ;IAClDC,MAAME,KAAK,GAAG;IACd,MAAM3B,gBAAgBwB,MAAM;QAC1B,MAAMA,KAAK2B,IAAI,CAAC,AAAC,CAAA,MAAMlD,WAAU,EAAGmD,WAAW;QAC/C,0FAA0F;QAC1F,gFAAgF;QAChF,MAAMrD,QAAQyB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,OAAO,eAAe0B,WAAW7B,IAAgB;IAC/C,MAAM0B,WAAW1B,MAAM,MAAMD,SAASC;AACxC;AAEA,OAAO,eAAe8B,eAAe9B,IAAgB;IACnD,MAAMC,QAAQ,MAAMF,SAASC;IAC7B,IAAIC,MAAME,KAAK,EAAE,MAAM,IAAIjC,WAAW,mBAAmB;IACzD,IAAI;QACF,MAAM8B,KAAK2B,IAAI,CAAC;QAChB,IAAI,CAAE,MAAMtB,eAAeL,OAAQ;YACjC,MAAM,IAAI9B,WAAW,mBAAmB;QAC1C;IACF,EAAE,OAAO6D,KAAK;QACZ,IAAIA,eAAe7D,YAAY,MAAM6D;QACrC,MAAM,IAAI7D,WAAW,mBAAmB,CAAC,8CAA8C,EAAE,AAAC6D,IAAcC,OAAO,CAAC,iDAAiD,CAAC;IACpK;AACF;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASC,kBAAkBC,GAAsB,EAAEvB,KAAe,EAAEwB,WAAqB,EAAEvB,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IACzJ,MAAMsB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAI5B,MAAMU,MAAM,GAAG,GAAG;QACpB,KAAK,MAAMmB,SAAS5D,OAAQ;YAC1BwD,WAAWd,IAAI,CAAC,GAAGzC,YAAY,CAAC2D,MAAM,CAAC,cAAc,EAAE7D,eAAeuD,KAAKM,OAAOL,YAAYd,MAAM,EAAE,IAAI,CAAC;YAC3GiB,YAAYhB,IAAI,IAAIa;QACtB;QACA,IAAIvB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMoB,aAAaP,IAAIQ,eAAe;YACtCL,UAAUf,IAAI,CAACmB;YACfF,WAAWjB,IAAI,CAACV,UAAU+B,GAAG,CAACxE,eAAeyE,IAAI,CAAC;QACpD;QACA,IAAI/B,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMoB,aAAaP,IAAIQ,eAAe;YACtC,MAAMG,WAAWhC,YAAY8B,GAAG,CAAC,CAACG,QAAU,CAAC,CAAC,EAAElE,OAAO+D,GAAG,CAAC,CAACH,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAEI,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJP,UAAUf,IAAI,CAAC,CAAC,CAAC,EAAEmB,WAAW,KAAK,EAAEI,SAAS,EAAE,CAAC;YACjDN,WAAWjB,IAAI,CAACT,YAAY8B,GAAG,CAACxE,eAAeyE,IAAI,CAAC;YACpDL,WAAWjB,IAAI,IAAIT,YAAYkC,OAAO,CAAC,CAACC,OAASpE,OAAO+D,GAAG,CAAC,IAAMxE,cAAc6E;QAClF;IACF;IAEA,KAAK,MAAMC,OAAOnC,WAAY;QAC5B,MAAMoC,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMX,SAAS5D,OAAQ;YAC1BwD,WAAWd,IAAI,CAAC,GAAGzC,YAAY,CAAC2D,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHF,YAAYhB,IAAI,CAACnD,cAAc+E;QACjC;QACAb,UAAUf,IAAI,CAAC,CAAC,CAAC,EAAE1C,OAAO+D,GAAG,CAAC,CAACH,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAEI,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GL,WAAWjB,IAAI,IAAI1C,OAAO+D,GAAG,CAAC,IAAMxE,cAAc+E;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWQ,IAAI,CAAC;QAAQS,SAAShB,UAAUO,IAAI,CAAC,YAAY;QAAQU,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAavD,IAAgB,EAAEX,KAAa,EAAEmE,IAAyB;IAC3F,MAAMC,cAAcrE,oBAAoBC;IACxC,IAAIoE,gBAAgB,MAAM;QACxB,MAAM,IAAIvF,WAAW,4BAA4B,CAAC,yCAAyC,EAAEuF,YAAYvE,KAAK,CAAC,GAAG,EAAEuE,YAAY/D,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAE+D,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE7C,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWrB;IACvF,IAAI2B,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMS,eAAe9B;IAC3C,MAAMkC,MAAM,MAAMzD;IAClB,MAAM0D,cAAcxB,MAAMU,MAAM,GAAG,IAAI,MAAM3C,iBAAiBsB,MAAMkC,KAAKvB,MAAMgC,GAAG,CAACxE,eAAeyE,IAAI,CAAC,QAAQ,EAAE;IAEjH,MAAM,EAAEQ,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGrB,kBAAkBC,KAAKvB,OAAOwB,aAAavB,WAAWC,aAAaC;IACzG,MAAMgD,eAAe/C,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAM0C,WAAWhD,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAM2C,MAAM,CAAC,kBAAkB,EAAEjD,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAE+B,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAMzD,OAAO,MAAMN,KAAKO,OAAO,CAACyD;IAChC,MAAMC,OAAQ,MAAM3D,KAAK4D,GAAG,IAAIZ,WAAYvC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACwC;KAAM;IAC7E,MAAMM,UAAUpD,QAAQM,MAAM,KAAK,IAAI4C,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQtD,QAAQuD,KAAK,CAAC,CAACnD,SAAWvC,OAAO2F,IAAI,CAAC,CAAC/B;oBAA8B6B;uBAApBjG,qBAAoBiG,aAAAA,GAAG,CAAC7B,MAAM,cAAV6B,wBAAAA,aAAc,IAAIlD;;IAC1J,OAAOgD,QAAQK,KAAK,CAAC,GAAGX,OAAOlB,GAAG,CAAC,CAAC,EAAE8B,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmB1E,IAAgB;IACjD,OAAO;QACL2E,OAAO,OAAOtF,OAAOmE,OAASD,aAAavD,MAAMX,OAAOmE;QACxDoB,WAAW,IAAMnE,iBAAiBT;IACpC;AACF"}

package/dist/esm/store/duckdb/native.d.ts

@@ -1,3 +1,6 @@
+import { type NativeModule } from '../native.js';
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
-export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.6-r.1";
-export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;
+type DuckdbApi = typeof import('@duckdb/node-api');
+export declare function duckdbNative(): Promise<NativeModule<DuckdbApi>>;
+export declare function duckdbApi(): Promise<DuckdbApi>;
+export {};

package/dist/esm/store/duckdb/native.js

@@ -1,16 +1,20 @@
-import { loadOrInstall, packageNodeModules } from '../native.js';
+import { loadOrInstall, packageNodeModules, peerRange } from '../native.js';
 export const DUCKDB_PACKAGE = '@duckdb/node-api';
-export const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@1.5.6-r.1`;
-const DUCKDB = {
-    store: 'duckdb',
-    pkg: DUCKDB_PACKAGE,
-    sizeHint: '~110MB',
-    installSpec: DUCKDB_INSTALL_SPEC
-};
-let duckdbApiPromise;
+let duckdbNativePromise;
 // The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning
 // the pre-install "not found" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.
-export function duckdbApi() {
-    if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall(DUCKDB, packageNodeModules());
-    return duckdbApiPromise;
+export function duckdbNative() {
+    if (!duckdbNativePromise) {
+        const descriptor = {
+            store: 'duckdb',
+            pkg: DUCKDB_PACKAGE,
+            sizeHint: '~110MB',
+            range: peerRange(DUCKDB_PACKAGE)
+        };
+        duckdbNativePromise = loadOrInstall(descriptor, packageNodeModules());
+    }
+    return duckdbNativePromise;
+}
+export async function duckdbApi() {
+    return (await duckdbNative()).api;
 }

package/dist/esm/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@1.5.6-r.1`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB_INSTALL_SPEC","DUCKDB","store","pkg","sizeHint","installSpec","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AACjD,OAAO,MAAMC,sBAAsB,GAAGD,eAAe,UAAU,CAAC,CAAC;AAEjE,MAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKJ;IACLK,UAAU;IACVC,aAAaL;AACf;AAEA,IAAIM;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBT,cAAiDI,QAAQH;IACnG,OAAOQ;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, type NativeModule, packageNodeModules, peerRange } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\ntype DuckdbApi = typeof import('@duckdb/node-api');\n\nlet duckdbNativePromise: Promise<NativeModule<DuckdbApi>> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbNative(): Promise<NativeModule<DuckdbApi>> {\n  if (!duckdbNativePromise) {\n    const descriptor: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB', range: peerRange(DUCKDB_PACKAGE) };\n    duckdbNativePromise = loadOrInstall<DuckdbApi>(descriptor, packageNodeModules());\n  }\n  return duckdbNativePromise;\n}\n\nexport async function duckdbApi(): Promise<DuckdbApi> {\n  return (await duckdbNative()).api;\n}\n"],"names":["loadOrInstall","packageNodeModules","peerRange","DUCKDB_PACKAGE","duckdbNativePromise","duckdbNative","descriptor","store","pkg","sizeHint","range","duckdbApi","api"],"mappings":"AAAA,SAASA,aAAa,EAA4CC,kBAAkB,EAAEC,SAAS,QAAQ,eAAe;AAEtH,OAAO,MAAMC,iBAAiB,mBAAmB;AAIjD,IAAIC;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,qBAAqB;QACxB,MAAME,aAA+B;YAAEC,OAAO;YAAUC,KAAKL;YAAgBM,UAAU;YAAUC,OAAOR,UAAUC;QAAgB;QAClIC,sBAAsBJ,cAAyBM,YAAYL;IAC7D;IACA,OAAOG;AACT;AAEA,OAAO,eAAeO;IACpB,OAAO,AAAC,CAAA,MAAMN,cAAa,EAAGO,GAAG;AACnC"}

package/dist/esm/store/duckdb/open.js

@@ -1,11 +1,13 @@
 import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures } from '../../features/index.js';
+import { ENGINE_CHECK_DOCS, engineIdentity, npmRemedy, checkEngine as runEngineCheck } from '../engine.js';
+import { peerRange } from '../native.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
-import { assertFtsReady, prepareFts } from './lexical.js';
-import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
+import { assertFtsReady, prepareFts, queryLexical } from './lexical.js';
+import { DUCKDB_PACKAGE, duckdbApi, duckdbNative } from './native.js';
 import { duckdbDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
@@ -89,6 +91,40 @@ async function connect(dbPath, _cfg, options) {
         throw err;
     }
 }
+async function duckdbEngineIdentity() {
+    return engineIdentity(DUCKDB_PACKAGE, (await duckdbNative()).version);
+}
+// The real schema, index build and lexical query over store/engine.ts's fixture, in memory.
+async function checkDuckdbEngine() {
+    const { api, version } = await duckdbNative();
+    const instance = await api.DuckDBInstance.create(':memory:');
+    const handle = {
+        instance,
+        duckdb: await instance.connect()
+    };
+    try {
+        const conn = createConnection(handle.duckdb);
+        await conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)');
+        await ensureCoreSchema(handle, conn);
+        await conn.runBatch('INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)', ENGINE_CHECK_DOCS.map((doc)=>[
+                doc.path,
+                doc.title,
+                doc.summary,
+                doc.text
+            ]));
+        await runEngineCheck('duckdb', engineIdentity(DUCKDB_PACKAGE, version), npmRemedy(DUCKDB_PACKAGE, peerRange(DUCKDB_PACKAGE)), async (terms)=>{
+            await prepareFts(conn);
+            return (await queryLexical(conn, terms, {
+                whereJoin: '',
+                whereCond: '',
+                scopeCond: '',
+                limit: 10
+            })).map((hit)=>hit.path);
+        });
+    } finally{
+        await close(handle);
+    }
+}
 // This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.
 export const duckdbOpenDialect = {
     filename: DB_FILENAME,
@@ -100,6 +136,8 @@ export const duckdbOpenDialect = {
     // way: posix "Could not set lock on file ... Conflicting lock is held in <exe> (PID n)", Windows
     // "Cannot open file ... being used by another process". Both are the same condition.
     isLocked: (err)=>/Could not set lock on file|being used by another process/.test(err.message),
+    engineIdentity: duckdbEngineIdentity,
+    checkEngine: checkDuckdbEngine,
     ensureCoreSchema,
     ensureFeatureSchema,
     prepareConnection,

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '6';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureCoreSchema(_handle: DuckdbHandle, conn: Connection): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\nasync function ensureFeatureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n}\n\nasync function prepareConnection(handle: DuckdbHandle, _conn: Connection, _cfg: Config): Promise<void> {\n  await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureCoreSchema,\n  ensureFeatureSchema,\n  prepareConnection,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["STORE_DIMS","SenseError","activeFeatures","openWithDialect","getMeta","setMeta","createConnection","assertFtsReady","prepareFts","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureCoreSchema","_handle","conn","exec","ensureFeatureSchema","cfg","feature","name","schema","prepareConnection","handle","_conn","_cfg","duckdb","close","disconnectSync","instance","closeSync","connect","dbPath","options","DuckDBInstance","err","message","create","observational","access_mode","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","prepareLexical","assertLexicalReady","openDuckdb"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,UAAU,QAAQ,eAAe;AAC1D,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,eAAeC,iBAAiBC,OAAqB,EAAEC,IAAgB;IACrE,MAAMA,KAAKC,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMD,KAAKC,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMD,KAAKC,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,MAAMD,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,IAAI,AAAC,MAAMhB,QAAQe,MAAM,sBAAuB,MAAM,MAAMd,QAAQc,MAAM,kBAAkBH;AAC9F;AAEA,eAAeK,oBAAoBH,OAAqB,EAAEC,IAAgB,EAAEG,GAAW;IACrF,KAAK,MAAMC,WAAWrB,eAAeoB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIC,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKC,IAAI,CAAC,CAAC,kKAAkK,EAAEpB,WAAW,+BAA+B,CAAC;YAChO;QACF;QACA,MAAMuB,QAAQE,MAAM,CAACN;IACvB;AACF;AAEA,eAAeO,kBAAkBC,MAAoB,EAAEC,KAAiB,EAAEC,IAAY;IACpF,MAAMf,uBAAuBa,OAAOG,MAAM,EAAE9B;AAC9C;AAEA,iFAAiF;AACjF,eAAe+B,MAAMJ,MAAoB;IACvCA,OAAOG,MAAM,CAACE,cAAc;IAC5BL,OAAOM,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEP,IAAoB,EAAEQ,OAA0D;IACrH,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAM5B,WAAU;IACxC,EAAE,OAAO6B,KAAK;QACZ,IAAIA,eAAetC,YAAY,MAAMsC;QACrC,MAAM,IAAItC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEQ,eAAe,UAAU,EAAE,AAAC8B,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIV;IACJ,IAAIG;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQC,QAAQK,aAAa,GAAG;YAAEC,aAAa;QAAY,IAAI;YAAEC,+BAA+B;QAAS;QAChJd,SAAS,MAAMG,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEmC,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAM5B,kBAAkBkB;QACxB,MAAMX,OAAOb,iBAAiBwB;QAC9B,OAAO;YAAEH,QAAQ;gBAAEM;gBAAUH;YAAO;YAAGX;QAAK;IAC9C,EAAE,OAAOoB,KAAK;QACZ,MAAMR,MAAM;YAAEE;YAAUH;QAAO;QAC/B,MAAMS;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMM,oBAA+C;IAC1DC,UAAU/B;IACVgC,eAAe/B;IACfgC,kBAAkBrC;IAClBwB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACV,MAAQ,2DAA2DW,IAAI,CAACX,IAAIC,OAAO;IAC9FvB;IACAI;IACAK;IACAyB,gBAAgB3C;IAChB4C,oBAAoB7C;IACpBM,aAAa,CAACc,QAAQR,OAASN,YAAYc,OAAOM,QAAQ,EAAEN,OAAOG,MAAM,EAAEX;AAC7E,EAAE;AAEF,OAAO,eAAekC,WAAW/B,GAAmB,EAAEe,OAA6B;IACjF,OAAOlC,gBAAgBmB,KAAKuB,mBAAmBR;AACjD"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport { ENGINE_CHECK_DOCS, engineIdentity, npmRemedy, checkEngine as runEngineCheck } from '../engine.ts';\nimport { peerRange } from '../native.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts, queryLexical } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi, duckdbNative } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '6';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureCoreSchema(_handle: DuckdbHandle, conn: Connection): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\nasync function ensureFeatureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n}\n\nasync function prepareConnection(handle: DuckdbHandle, _conn: Connection, _cfg: Config): Promise<void> {\n  await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\nasync function duckdbEngineIdentity(): Promise<string> {\n  return engineIdentity(DUCKDB_PACKAGE, (await duckdbNative()).version);\n}\n\n// The real schema, index build and lexical query over store/engine.ts's fixture, in memory.\nasync function checkDuckdbEngine(): Promise<void> {\n  const { api, version } = await duckdbNative();\n  const instance = await api.DuckDBInstance.create(':memory:');\n  const handle = { instance, duckdb: await instance.connect() };\n  try {\n    const conn = createConnection(handle.duckdb);\n    await conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)');\n    await ensureCoreSchema(handle, conn);\n    await conn.runBatch(\n      'INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)',\n      ENGINE_CHECK_DOCS.map((doc) => [doc.path, doc.title, doc.summary, doc.text])\n    );\n    await runEngineCheck('duckdb', engineIdentity(DUCKDB_PACKAGE, version), npmRemedy(DUCKDB_PACKAGE, peerRange(DUCKDB_PACKAGE)), async (terms) => {\n      await prepareFts(conn);\n      return (await queryLexical(conn, terms, { whereJoin: '', whereCond: '', scopeCond: '', limit: 10 })).map((hit) => hit.path);\n    });\n  } finally {\n    await close(handle);\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  engineIdentity: duckdbEngineIdentity,\n  checkEngine: checkDuckdbEngine,\n  ensureCoreSchema,\n  ensureFeatureSchema,\n  prepareConnection,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["STORE_DIMS","SenseError","activeFeatures","ENGINE_CHECK_DOCS","engineIdentity","npmRemedy","checkEngine","runEngineCheck","peerRange","openWithDialect","getMeta","setMeta","createConnection","assertFtsReady","prepareFts","queryLexical","DUCKDB_PACKAGE","duckdbApi","duckdbNative","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureCoreSchema","_handle","conn","exec","ensureFeatureSchema","cfg","feature","name","schema","prepareConnection","handle","_conn","_cfg","duckdb","close","disconnectSync","instance","closeSync","connect","dbPath","options","DuckDBInstance","err","message","create","observational","access_mode","storage_compatibility_version","duckdbEngineIdentity","version","checkDuckdbEngine","api","runBatch","map","doc","path","title","summary","text","terms","whereJoin","whereCond","scopeCond","limit","hit","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","prepareLexical","assertLexicalReady","openDuckdb"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AACzD,SAASC,iBAAiB,EAAEC,cAAc,EAAEC,SAAS,EAAEC,eAAeC,cAAc,QAAQ,eAAe;AAC3G,SAASC,SAAS,QAAQ,eAAe;AAEzC,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,UAAU,EAAEC,YAAY,QAAQ,eAAe;AACxE,SAASC,cAAc,EAAEC,SAAS,EAAEC,YAAY,QAAQ,cAAc;AACtE,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,eAAeC,iBAAiBC,OAAqB,EAAEC,IAAgB;IACrE,MAAMA,KAAKC,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMD,KAAKC,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMD,KAAKC,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,MAAMD,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,IAAI,AAAC,MAAMlB,QAAQiB,MAAM,sBAAuB,MAAM,MAAMhB,QAAQgB,MAAM,kBAAkBH;AAC9F;AAEA,eAAeK,oBAAoBH,OAAqB,EAAEC,IAAgB,EAAEG,GAAW;IACrF,KAAK,MAAMC,WAAW7B,eAAe4B,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIC,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKC,IAAI,CAAC,CAAC,kKAAkK,EAAE5B,WAAW,+BAA+B,CAAC;YAChO;QACF;QACA,MAAM+B,QAAQE,MAAM,CAACN;IACvB;AACF;AAEA,eAAeO,kBAAkBC,MAAoB,EAAEC,KAAiB,EAAEC,IAAY;IACpF,MAAMf,uBAAuBa,OAAOG,MAAM,EAAEtC;AAC9C;AAEA,iFAAiF;AACjF,eAAeuC,MAAMJ,MAAoB;IACvCA,OAAOG,MAAM,CAACE,cAAc;IAC5BL,OAAOM,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEP,IAAoB,EAAEQ,OAA0D;IACrH,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAM7B,WAAU;IACxC,EAAE,OAAO8B,KAAK;QACZ,IAAIA,eAAe9C,YAAY,MAAM8C;QACrC,MAAM,IAAI9C,WAAW,4BAA4B,CAAC,yBAAyB,EAAEe,eAAe,UAAU,EAAE,AAAC+B,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIV;IACJ,IAAIG;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQC,QAAQK,aAAa,GAAG;YAAEC,aAAa;QAAY,IAAI;YAAEC,+BAA+B;QAAS;QAChJd,SAAS,MAAMG,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAIzC,WAAW,4BAA4B,CAAC,8BAA8B,EAAE2C,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAM5B,kBAAkBkB;QACxB,MAAMX,OAAOf,iBAAiB0B;QAC9B,OAAO;YAAEH,QAAQ;gBAAEM;gBAAUH;YAAO;YAAGX;QAAK;IAC9C,EAAE,OAAOoB,KAAK;QACZ,MAAMR,MAAM;YAAEE;YAAUH;QAAO;QAC/B,MAAMS;IACR;AACF;AAEA,eAAeM;IACb,OAAOjD,eAAeY,gBAAgB,AAAC,CAAA,MAAME,cAAa,EAAGoC,OAAO;AACtE;AAEA,4FAA4F;AAC5F,eAAeC;IACb,MAAM,EAAEC,GAAG,EAAEF,OAAO,EAAE,GAAG,MAAMpC;IAC/B,MAAMuB,WAAW,MAAMe,IAAIV,cAAc,CAACG,MAAM,CAAC;IACjD,MAAMd,SAAS;QAAEM;QAAUH,QAAQ,MAAMG,SAASE,OAAO;IAAG;IAC5D,IAAI;QACF,MAAMhB,OAAOf,iBAAiBuB,OAAOG,MAAM;QAC3C,MAAMX,KAAKC,IAAI,CAAC;QAChB,MAAMH,iBAAiBU,QAAQR;QAC/B,MAAMA,KAAK8B,QAAQ,CACjB,0EACAtD,kBAAkBuD,GAAG,CAAC,CAACC,MAAQ;gBAACA,IAAIC,IAAI;gBAAED,IAAIE,KAAK;gBAAEF,IAAIG,OAAO;gBAAEH,IAAII,IAAI;aAAC;QAE7E,MAAMxD,eAAe,UAAUH,eAAeY,gBAAgBsC,UAAUjD,UAAUW,gBAAgBR,UAAUQ,kBAAkB,OAAOgD;YACnI,MAAMlD,WAAWa;YACjB,OAAO,AAAC,CAAA,MAAMZ,aAAaY,MAAMqC,OAAO;gBAAEC,WAAW;gBAAIC,WAAW;gBAAIC,WAAW;gBAAIC,OAAO;YAAG,EAAC,EAAGV,GAAG,CAAC,CAACW,MAAQA,IAAIT,IAAI;QAC5H;IACF,SAAU;QACR,MAAMrB,MAAMJ;IACd;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMmC,oBAA+C;IAC1DC,UAAUhD;IACViD,eAAehD;IACfiD,kBAAkBtD;IAClBwB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFmC,UAAU,CAAC3B,MAAQ,2DAA2D4B,IAAI,CAAC5B,IAAIC,OAAO;IAC9F5C,gBAAgBiD;IAChB/C,aAAaiD;IACb9B;IACAI;IACAK;IACA0C,gBAAgB9D;IAChB+D,oBAAoBhE;IACpBQ,aAAa,CAACc,QAAQR,OAASN,YAAYc,OAAOM,QAAQ,EAAEN,OAAOG,MAAM,EAAEX;AAC7E,EAAE;AAEF,OAAO,eAAemD,WAAWhD,GAAmB,EAAEe,OAA6B;IACjF,OAAOpC,gBAAgBqB,KAAKwC,mBAAmBzB;AACjD"}

package/dist/esm/store/duckdb/ordered-bm25.d.ts

@@ -1,4 +1,4 @@
 import type { Connection } from '../types.js';
-export declare const FTS_STEMMER = "porter";
-export declare function nativeQueryTerms(conn: Connection, query: string): Promise<string[]>;
-export declare function orderedBm25Sql(field: 'title' | 'summary' | 'text', termCount: number): string;
+import type { DuckdbFtsBuilders } from './engine.js';
+export declare function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]>;
+export declare function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string;

package/dist/esm/store/duckdb/ordered-bm25.js

@@ -1,20 +1,15 @@
-export const FTS_STEMMER = 'porter';
 // Keep deduplication native, but return original tokens so native scoring stems once.
-export async function nativeQueryTerms(conn, query) {
-    const stmt = await conn.prepare(`
-    WITH tokens AS (SELECT unnest(fts_main_content.tokenize(?)) AS raw_term)
-    SELECT min(raw_term) AS term FROM tokens
-    WHERE raw_term <> '' GROUP BY stem(raw_term, '${FTS_STEMMER}') ORDER BY term
-  `);
+export async function nativeQueryTerms(conn, fts, query) {
+    const stmt = await conn.prepare(fts.queryTerms());
     const rows = await stmt.all(query);
     return rows.map(({ term })=>term);
 }
 // Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.
 // Only the number of terms shapes this authored SQL; query values are always bound.
-export function orderedBm25Sql(field, termCount) {
+export function orderedBm25Sql(fts, field, termCount) {
     if (termCount === 0) return 'NULL';
     const contributions = Array.from({
         length: termCount
-    }, ()=>`SELECT fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false) AS score`);
+    }, ()=>fts.termScore(field));
     return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;
 }

package/dist/esm/store/duckdb/ordered-bm25.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import type { Connection } from '../types.ts';\n\nexport const FTS_STEMMER = 'porter';\n\n// Keep deduplication native, but return original tokens so native scoring stems once.\nexport async function nativeQueryTerms(conn: Connection, query: string): Promise<string[]> {\n  const stmt = await conn.prepare(`\n    WITH tokens AS (SELECT unnest(fts_main_content.tokenize(?)) AS raw_term)\n    SELECT min(raw_term) AS term FROM tokens\n    WHERE raw_term <> '' GROUP BY stem(raw_term, '${FTS_STEMMER}') ORDER BY term\n  `);\n  const rows = (await stmt.all(query)) as Array<{ term: string }>;\n  return rows.map(({ term }) => term);\n}\n\n// Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.\n// Only the number of terms shapes this authored SQL; query values are always bound.\nexport function orderedBm25Sql(field: 'title' | 'summary' | 'text', termCount: number): string {\n  if (termCount === 0) return 'NULL';\n  const contributions = Array.from({ length: termCount }, () => `SELECT fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false) AS score`);\n  return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;\n}\n"],"names":["FTS_STEMMER","nativeQueryTerms","conn","query","stmt","prepare","rows","all","map","term","orderedBm25Sql","field","termCount","contributions","Array","from","length","join"],"mappings":"AAEA,OAAO,MAAMA,cAAc,SAAS;AAEpC,sFAAsF;AACtF,OAAO,eAAeC,iBAAiBC,IAAgB,EAAEC,KAAa;IACpE,MAAMC,OAAO,MAAMF,KAAKG,OAAO,CAAC,CAAC;;;kDAGe,EAAEL,YAAY;EAC9D,CAAC;IACD,MAAMM,OAAQ,MAAMF,KAAKG,GAAG,CAACJ;IAC7B,OAAOG,KAAKE,GAAG,CAAC,CAAC,EAAEC,IAAI,EAAE,GAAKA;AAChC;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,SAASC,eAAeC,KAAmC,EAAEC,SAAiB;IACnF,IAAIA,cAAc,GAAG,OAAO;IAC5B,MAAMC,gBAAgBC,MAAMC,IAAI,CAAC;QAAEC,QAAQJ;IAAU,GAAG,IAAM,CAAC,iEAAiE,EAAED,MAAM,iCAAiC,CAAC;IAC1K,OAAO,CAAC,wCAAwC,EAAEE,cAAcI,IAAI,CAAC,eAAe,EAAE,CAAC;AACzF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import type { Connection } from '../types.ts';\nimport type { DuckdbFtsBuilders } from './engine.ts';\n\n// Keep deduplication native, but return original tokens so native scoring stems once.\nexport async function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]> {\n  const stmt = await conn.prepare(fts.queryTerms());\n  const rows = (await stmt.all(query)) as Array<{ term: string }>;\n  return rows.map(({ term }) => term);\n}\n\n// Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.\n// Only the number of terms shapes this authored SQL; query values are always bound.\nexport function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string {\n  if (termCount === 0) return 'NULL';\n  const contributions = Array.from({ length: termCount }, () => fts.termScore(field));\n  return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;\n}\n"],"names":["nativeQueryTerms","conn","fts","query","stmt","prepare","queryTerms","rows","all","map","term","orderedBm25Sql","field","termCount","contributions","Array","from","length","termScore","join"],"mappings":"AAGA,sFAAsF;AACtF,OAAO,eAAeA,iBAAiBC,IAAgB,EAAEC,GAAsB,EAAEC,KAAa;IAC5F,MAAMC,OAAO,MAAMH,KAAKI,OAAO,CAACH,IAAII,UAAU;IAC9C,MAAMC,OAAQ,MAAMH,KAAKI,GAAG,CAACL;IAC7B,OAAOI,KAAKE,GAAG,CAAC,CAAC,EAAEC,IAAI,EAAE,GAAKA;AAChC;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,SAASC,eAAeT,GAAsB,EAAEU,KAAmC,EAAEC,SAAiB;IAC3G,IAAIA,cAAc,GAAG,OAAO;IAC5B,MAAMC,gBAAgBC,MAAMC,IAAI,CAAC;QAAEC,QAAQJ;IAAU,GAAG,IAAMX,IAAIgB,SAAS,CAACN;IAC5E,OAAO,CAAC,wCAAwC,EAAEE,cAAcK,IAAI,CAAC,eAAe,EAAE,CAAC;AACzF"}

package/dist/esm/store/engine.d.ts

@@ -0,0 +1,28 @@
+import { SenseError } from '../errors.js';
+export declare const ENGINE_IDENTITY_META_KEY = "engine_identity";
+export declare function engineIdentity(pkg: string, version: string | undefined): string;
+/**
+ * One version-specific replacement of a store's SQL builders. `range` matches the engine version (the npm package's,
+ * or Node's bundled SQLite for sqlite), prereleases included since every `@duckdb/node-api` release is one;
+ * `reason` names the observed difference.
+ */
+export interface EnginePatch<Builders> {
+    range: string;
+    reason: string;
+    builders: Partial<Builders>;
+}
+export declare function selectBuilders<Builders extends object>(defaults: Builders, patches: readonly EnginePatch<Builders>[], version: string | undefined): Builders;
+export interface EngineCheckDoc {
+    path: string;
+    title: string;
+    summary: string;
+    text: string;
+}
+export declare const ENGINE_CHECK_DOCS: readonly EngineCheckDoc[];
+export declare const ENGINE_CHECK_QUERIES: ReadonlyArray<{
+    terms: string;
+    paths: string[];
+}>;
+export declare function npmRemedy(pkg: string, range: string): string;
+export declare function checkEngine(store: string, identity: string, remedy: string, query: (terms: string) => Promise<string[]>): Promise<void>;
+export declare function engineCheckFailed(store: string, identity: string, remedy: string, failure: string): SenseError;