sensemaking 0.18.3 → 0.19.0

package/dist/esm/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by\n// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with\n// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --\n// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own\n// representation and stays as it is. Every function here takes `dims` as a parameter rather\n// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real\n// constant at the call site).\n//\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only\n// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's\n// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever\n// open.ts already resolved, including via an on-demand install (see native.ts).\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced\n// under the column's width); zero-padding leaves cosine scores unchanged since the added\n// dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces\n// for every store, see toStore); dequantizing into a plain float array is this store's own\n// representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's\n// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native\n// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --\n// automatic type inference on the raw values misreads an all-integer first component (a real\n// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row\n// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks\n// the winning chunk's line range from the same row max(score) came from, per path. An empty\n// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed\n// into one native scan: the sampled target vectors become a small VALUES list, cross-joined\n// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost\n// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","score","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,mFAAmF;AACnF,uFAAuF;AACvF,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAC5F,8FAA8F;AAC9F,8BAA8B;AAC9B,EAAE;AACF,4FAA4F;AAC5F,8FAA8F;AAC9F,gGAAgG;AAChG,gFAAgF;AAEhF,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,6FAA6F;AAC7F,yFAAyF;AACzF,wEAAwE;AACxE,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,2FAA2F;AAC3F,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,0FAA0F;AAC1F,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,6FAA6F;AAC7F,4FAA4F;AAC5F,2FAA2F;AAC3F,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;YAAC4B,WAAW7B,OAAO6C,IAAI3C;eAAW+C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ7B;eAAU+C,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC7FoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACjH,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,8FAA8F;AAC9F,wFAAwF;AACxF,OAAO,eAAeiB,YAAYlC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAEsB,IAAgE;IACxK,IAAIA,KAAKd,OAAO,IAAIc,KAAKd,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMc,aAAa,MAAMnC,KAAKQ,OAAO,CAAC;IACtC,MAAM4B,aAAc,MAAMD,WAAWE,GAAG,CAACzB;IACzC,IAAIwB,WAAWtD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUtE,aAAaoE,YAAYb,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMqE,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMlB,cAAcY,KAAKd,OAAO,GAAG;WAAIc,KAAKd,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE6B,QAAQf,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEoD,YAAYzD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYwD,YAAYzD,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEoD,KAAKd,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIgE,QAAQf,GAAG,CAAC,CAACkB,IAAMvC,WAAW7B,OAAOoE,GAAGlE;YAASqC;eAAS2B;eAAgBjB;YAAaY,KAAKQ,CAAC;SAAC;QAChI,MAAMhC,QAAsB;eAAI4B,QAAQf,GAAG,CAAC,IAAMpB,MAAMC,QAAQ7B;YAAQJ;eAAYoE,YAAYhB,GAAG,CAAC,IAAMpD;eAAamD,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACxE,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks the winning chunk's line range from the same row max(score) came from, per path.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","score","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,oIAAoI;AACpI,8QAA8Q;AAC9Q,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,8KAA8K;AAC9K,oGAAoG;AACpG,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;YAAC4B,WAAW7B,OAAO6C,IAAI3C;eAAW+C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ7B;eAAU+C,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC7FoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACjH,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeiB,YAAYlC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAEsB,IAAgE;IACxK,IAAIA,KAAKd,OAAO,IAAIc,KAAKd,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMc,aAAa,MAAMnC,KAAKQ,OAAO,CAAC;IACtC,MAAM4B,aAAc,MAAMD,WAAWE,GAAG,CAACzB;IACzC,IAAIwB,WAAWtD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUtE,aAAaoE,YAAYb,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMqE,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMlB,cAAcY,KAAKd,OAAO,GAAG;WAAIc,KAAKd,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE6B,QAAQf,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEoD,YAAYzD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYwD,YAAYzD,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEoD,KAAKd,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIgE,QAAQf,GAAG,CAAC,CAACkB,IAAMvC,WAAW7B,OAAOoE,GAAGlE;YAASqC;eAAS2B;eAAgBjB;YAAaY,KAAKQ,CAAC;SAAC;QAChI,MAAMhC,QAAsB;eAAI4B,QAAQf,GAAG,CAAC,IAAMpB,MAAMC,QAAQ7B;YAAQJ;eAAYoE,YAAYhB,GAAG,CAAC,IAAMpD;eAAamD,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACxE,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF"}

package/dist/esm/store/index.d.ts

@@ -2,7 +2,8 @@ import type { ResolvedConfig } from '../config/index.js';
 import type { OpenResult } from './sqlite/open.js';
 export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
 export declare function requireWatchConcurrency(cfg: ResolvedConfig): void;
-export { getMeta, setMeta } from './meta.js';
+export { clearCache } from './cache.js';
+export { getMeta, setMeta } from './shared.js';
 export type { OpenResult } from './sqlite/open.js';
-export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';

package/dist/esm/store/index.js

@@ -32,10 +32,8 @@ function entryFor(name) {
 export async function openStore(cfg) {
     const name = storeName(cfg);
     const entry = entryFor(name);
-    // The one unambiguous, config-level capability need: an embed block some preset actually
-    // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level
-    // (every preset defaults to wanting "words"), so a store lacking 'lexical' fails loudly at
-    // first use instead (see duckdb/store.ts's lexical.query()).
+    // The one unambiguous, config-level capability need: an embed block some preset actually uses
+    // for vectors. Lexical search has no such signal at this level (every preset defaults to "words"), so a store lacking it fails loudly at first use.
     if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "vectors" in this build, but this config's "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)`);
     }
@@ -49,5 +47,6 @@ export function requireWatchConcurrency(cfg) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "watch-concurrency" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set "store" to "sqlite" in sense.config.json`);
     }
 }
-export { getMeta, setMeta } from './meta.js';
-export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { clearCache } from './cache.js';
+export { getMeta, setMeta } from './shared.js';
+export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually\n  // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level\n  // (every preset defaults to wanting \"words\"), so a store lacking 'lexical' fails loudly at\n  // first use instead (see duckdb/store.ts's lexical.query()).\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { getMeta, setMeta } from './meta.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","getMeta","setMeta","clearCache","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,6DAA6D;IAC7D,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,OAAO,EAAEC,OAAO,QAAQ,YAAY;AAE7C,SAASC,UAAU,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","clearCache","getMeta","setMeta","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AACxC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,SAASC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}

package/dist/esm/store/native.js

@@ -3,17 +3,12 @@ import Module from 'node:module';
 import { dirname, join } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import { SenseError } from '../errors.js';
-// install-module-linked itself is small, but a target native package can be large (@duckdb/node-
-// api's ~110MB is the current instance): deferred the same way embed/static.ts defers
-// @huggingface/tokenizers, so requiring it stays inside the already-gated branch below rather
-// than running for every store/index.ts load.
+// install-module-linked is small, but a target native package can be large (@duckdb/node-api's
+// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.
 const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
 const _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));
-// Walks up from this file's own directory to find the real package.json, rather than a fixed
-// dot-count from __dirname (xz-compat's `../../node_modules`): this file sits at a different
-// depth under src/ during `tsds test:node` than under the built dist/{cjs,esm}/store output, so a
-// hardcoded depth is right for one and wrong for the other. Works the same way for a
-// globally-installed `sense`, whose node_modules this resolves to -- never the caller's cwd.
+// Walks up from this file's own directory rather than a fixed dot-count: it sits at a
+// different depth under src/ than under built dist/{cjs,esm}/store output.
 export function packageNodeModules() {
     let dir = _dirname;
     for(;;){
@@ -28,12 +23,8 @@ export function packageNodeModules() {
     }
 }
 const installing = new Map();
-// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).
-// The install here is async -- the load site already awaits a dynamic import -- so a bare
-// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target
-// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,
-// keyed by specifier and target so tests exercising several installs in one process don't share
-// one slot. A failed install is not cached, so a later call retries.
+// De-dupes concurrent installs: since the install is async, concurrent callers await the same
+// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.
 function installOnce(specifier, nodeModulesPath) {
     const key = `${specifier}\n${nodeModulesPath}`;
     let promise = installing.get(key);
@@ -47,12 +38,8 @@ function installOnce(specifier, nodeModulesPath) {
     }
     return promise;
 }
-// Node caches a bare specifier's package resolution -- including a "not found" miss -- for the
-// life of the process, so retrying `import(importName)` right after installing sees the same
-// stale miss and falls through to a bogus index.js guess instead of the package's real entry
-// (verified against a real global install: the plain-specifier retry failed there, this did
-// not). Reading the freshly-installed package's own package.json and importing its entry file
-// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.
+// Node caches a bare specifier's resolution, including a "not found" miss, for the process
+// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.
 function resolveEntryUrl(importName, nodeModulesPath) {
     var _pkg_main;
     const pkgDir = join(nodeModulesPath, ...importName.split('/'));
@@ -60,10 +47,8 @@ function resolveEntryUrl(importName, nodeModulesPath) {
     const entry = typeof pkg.exports === 'string' ? pkg.exports : (_pkg_main = pkg.main) !== null && _pkg_main !== void 0 ? _pkg_main : 'index.js';
     return pathToFileURL(join(pkgDir, entry)).href;
 }
-// Import-then-install-then-retry, over an injectable importName so the failure path (import AND
-// install both fail) can be exercised for real in tests without downloading a real store's native
-// binding. importName defaults to descriptor.pkg, which is what production always wants; the
-// thrown errors name descriptor.pkg regardless, since that is what a real user needs to install.
+// Import-then-install-then-retry, over an injectable importName so the failure path can be
+// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.
 export async function loadOrInstall(descriptor, nodeModulesPath, importName = descriptor.pkg) {
     try {
         return await import(importName);

package/dist/esm/store/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked itself is small, but a target native package can be large (@duckdb/node-\n// api's ~110MB is the current instance): deferred the same way embed/static.ts defers\n// @huggingface/tokenizers, so requiring it stays inside the already-gated branch below rather\n// than running for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory to find the real package.json, rather than a fixed\n// dot-count from __dirname (xz-compat's `../../node_modules`): this file sits at a different\n// depth under src/ during `tsds test:node` than under the built dist/{cjs,esm}/store output, so a\n// hardcoded depth is right for one and wrong for the other. Works the same way for a\n// globally-installed `sense`, whose node_modules this resolves to -- never the caller's cwd.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).\n// The install here is async -- the load site already awaits a dynamic import -- so a bare\n// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target\n// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,\n// keyed by specifier and target so tests exercising several installs in one process don't share\n// one slot. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's package resolution -- including a \"not found\" miss -- for the\n// life of the process, so retrying `import(importName)` right after installing sees the same\n// stale miss and falls through to a bogus index.js guess instead of the package's real entry\n// (verified against a real global install: the plain-specifier retry failed there, this did\n// not). Reading the freshly-installed package's own package.json and importing its entry file\n// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path (import AND\n// install both fail) can be exercised for real in tests without downloading a real store's native\n// binding. importName defaults to descriptor.pkg, which is what production always wants; the\n// thrown errors name descriptor.pkg regardless, since that is what a real user needs to install.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,iGAAiG;AACjG,sFAAsF;AACtF,8FAA8F;AAC9F,8CAA8C;AAC9C,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,6FAA6F;AAC7F,6FAA6F;AAC7F,kGAAkG;AAClG,qFAAqF;AACrF,6FAA6F;AAC7F,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,qEAAqE;AACrE,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,6FAA6F;AAC7F,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOX,cAAcF,KAAKmC,QAAQE,QAAQG,IAAI;AAChD;AAEA,gGAAgG;AAChG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,OAAO,eAAeC,cAAiBC,UAA4B,EAAEnB,eAAuB,EAAEW,aAAqBQ,WAAW7B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNS,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,wCAAwC,EAAE6B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAMzB,YAAYa,YAAYX;QAChC,EAAE,OAAOwB,YAAY;YACnB,MAAM,IAAI5C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,0CAA0C,EAAE,AAACkC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW7B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,gBAAgBC,YAAYX;QACnD,EAAE,OAAO0B,SAAS;YAChB,MAAM,IAAI9C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW7B,GAAG,CAAC,6BAA6B,EAAE,AAACoC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW7B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,+FAA+F;AAC/F,sGAAsG;AACtG,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,sFAAsF;AACtF,2EAA2E;AAC3E,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,kGAAkG;AAClG,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOX,cAAcF,KAAKmC,QAAQE,QAAQG,IAAI;AAChD;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,OAAO,eAAeC,cAAiBC,UAA4B,EAAEnB,eAAuB,EAAEW,aAAqBQ,WAAW7B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNS,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,wCAAwC,EAAE6B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAMzB,YAAYa,YAAYX;QAChC,EAAE,OAAOwB,YAAY;YACnB,MAAM,IAAI5C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,0CAA0C,EAAE,AAACkC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW7B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,gBAAgBC,YAAYX;QACnD,EAAE,OAAO0B,SAAS;YAChB,MAAM,IAAI9C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW7B,GAAG,CAAC,6BAA6B,EAAE,AAACoC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW7B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}

package/dist/esm/store/sql-functions.js

@@ -1,7 +1,6 @@
 import posix from 'node:path/posix';
 // has(field, value) / basename(path, suffix): the two sense-supplied SQL functions with no
-// engine-specific pieces, shared by sqlite/sql-functions.ts and duckdb/sql-functions.ts so each
-// store registers the identical implementation instead of a hand-copied one (DRY).
+// engine-specific pieces, registered identically by each store instead of a hand-copied one.
 // JSON-array field -> membership, string field -> substring, NULL -> false.
 export function hasImpl(field, value) {
     if (field === null || field === undefined) return 0;
@@ -17,9 +16,8 @@ export function hasImpl(field, value) {
     }
     return String(field).includes(needle) ? 1 : 0;
 }
-// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither
-// engine has a filename function of its own, and LIKE tricks that fake one also match folder
-// names.
+// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither engine
+// has a filename function of its own, and LIKE tricks that fake one also match folder names.
 export function basenameImpl(path, suffix) {
     if (path === null || path === undefined) return null;
     const name = posix.basename(String(path));

package/dist/esm/store/sql-functions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sql-functions.ts"],"sourcesContent":["import posix from 'node:path/posix';\n\n// has(field, value) / basename(path, suffix): the two sense-supplied SQL functions with no\n// engine-specific pieces, shared by sqlite/sql-functions.ts and duckdb/sql-functions.ts so each\n// store registers the identical implementation instead of a hand-copied one (DRY).\n\n// JSON-array field -> membership, string field -> substring, NULL -> false.\nexport function hasImpl(field: unknown, value: unknown): number {\n  if (field === null || field === undefined) return 0;\n  const needle = String(value);\n  if (typeof field === 'string') {\n    if (field.startsWith('[')) {\n      try {\n        const parsed = JSON.parse(field);\n        if (Array.isArray(parsed)) return parsed.some((item) => String(item) === needle) ? 1 : 0;\n      } catch {}\n    }\n    return field.includes(needle) ? 1 : 0;\n  }\n  return String(field).includes(needle) ? 1 : 0;\n}\n\n// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither\n// engine has a filename function of its own, and LIKE tricks that fake one also match folder\n// names.\nexport function basenameImpl(path: unknown, suffix?: unknown): string | null {\n  if (path === null || path === undefined) return null;\n  const name = posix.basename(String(path));\n  const tail = suffix === null || suffix === undefined ? '' : String(suffix);\n  // POSIX: a suffix identical to the whole name is not removed (node's own basename strips it).\n  return tail !== '' && tail !== name && name.endsWith(tail) ? name.slice(0, -tail.length) : name;\n}\n"],"names":["posix","hasImpl","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","basenameImpl","path","suffix","name","basename","tail","endsWith","slice","length"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,2FAA2F;AAC3F,gGAAgG;AAChG,mFAAmF;AAEnF,4EAA4E;AAC5E,OAAO,SAASC,QAAQC,KAAc,EAAEC,KAAc;IACpD,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;IAClD,MAAMC,SAASC,OAAOH;IACtB,IAAI,OAAOD,UAAU,UAAU;QAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;YACzB,IAAI;gBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;gBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;YACzF,EAAE,OAAM,CAAC;QACX;QACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;IACtC;IACA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;AAC9C;AAEA,0FAA0F;AAC1F,6FAA6F;AAC7F,SAAS;AACT,OAAO,SAASW,aAAaC,IAAa,EAAEC,MAAgB;IAC1D,IAAID,SAAS,QAAQA,SAASb,WAAW,OAAO;IAChD,MAAMe,OAAOnB,MAAMoB,QAAQ,CAACd,OAAOW;IACnC,MAAMI,OAAOH,WAAW,QAAQA,WAAWd,YAAY,KAAKE,OAAOY;IACnE,8FAA8F;IAC9F,OAAOG,SAAS,MAAMA,SAASF,QAAQA,KAAKG,QAAQ,CAACD,QAAQF,KAAKI,KAAK,CAAC,GAAG,CAACF,KAAKG,MAAM,IAAIL;AAC7F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sql-functions.ts"],"sourcesContent":["import posix from 'node:path/posix';\n\n// has(field, value) / basename(path, suffix): the two sense-supplied SQL functions with no\n// engine-specific pieces, registered identically by each store instead of a hand-copied one.\n\n// JSON-array field -> membership, string field -> substring, NULL -> false.\nexport function hasImpl(field: unknown, value: unknown): number {\n  if (field === null || field === undefined) return 0;\n  const needle = String(value);\n  if (typeof field === 'string') {\n    if (field.startsWith('[')) {\n      try {\n        const parsed = JSON.parse(field);\n        if (Array.isArray(parsed)) return parsed.some((item) => String(item) === needle) ? 1 : 0;\n      } catch {}\n    }\n    return field.includes(needle) ? 1 : 0;\n  }\n  return String(field).includes(needle) ? 1 : 0;\n}\n\n// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither engine\n// has a filename function of its own, and LIKE tricks that fake one also match folder names.\nexport function basenameImpl(path: unknown, suffix?: unknown): string | null {\n  if (path === null || path === undefined) return null;\n  const name = posix.basename(String(path));\n  const tail = suffix === null || suffix === undefined ? '' : String(suffix);\n  // POSIX: a suffix identical to the whole name is not removed (node's own basename strips it).\n  return tail !== '' && tail !== name && name.endsWith(tail) ? name.slice(0, -tail.length) : name;\n}\n"],"names":["posix","hasImpl","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","basenameImpl","path","suffix","name","basename","tail","endsWith","slice","length"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,2FAA2F;AAC3F,6FAA6F;AAE7F,4EAA4E;AAC5E,OAAO,SAASC,QAAQC,KAAc,EAAEC,KAAc;IACpD,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;IAClD,MAAMC,SAASC,OAAOH;IACtB,IAAI,OAAOD,UAAU,UAAU;QAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;YACzB,IAAI;gBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;gBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;YACzF,EAAE,OAAM,CAAC;QACX;QACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;IACtC;IACA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;AAC9C;AAEA,iGAAiG;AACjG,6FAA6F;AAC7F,OAAO,SAASW,aAAaC,IAAa,EAAEC,MAAgB;IAC1D,IAAID,SAAS,QAAQA,SAASb,WAAW,OAAO;IAChD,MAAMe,OAAOnB,MAAMoB,QAAQ,CAACd,OAAOW;IACnC,MAAMI,OAAOH,WAAW,QAAQA,WAAWd,YAAY,KAAKE,OAAOY;IACnE,8FAA8F;IAC9F,OAAOG,SAAS,MAAMA,SAASF,QAAQA,KAAKG,QAAQ,CAACD,QAAQF,KAAKI,KAAK,CAAC,GAAG,CAACF,KAAKG,MAAM,IAAIL;AAC7F"}

package/dist/esm/store/sqlite/connection.js

@@ -1,8 +1,6 @@
 import { withTransaction } from '../transaction.js';
-// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every
-// store presents. Every method below does its real work synchronously and returns an already-
-// resolved promise, so reconcile's loop, the feature hooks, and runBatch never actually cross
-// an async boundary internally -- no regression against the pre-Store synchronous code.
+// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real
+// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.
 let SqliteStatement = class SqliteStatement {
     async run(...params) {
         return this.stmt.run(...params);
@@ -34,10 +32,8 @@ export function createConnection(db) {
         async prepare (sql) {
             return new SqliteStatement(db.prepare(sql));
         },
-        // Prepares once and loops synchronously: today's code, unchanged, wrapped as one async call
-        // so features share one Connection contract with DuckDB. Joins the caller's transaction when
-        // there is one (reconcile); otherwise opens its own, so a standalone batch (a vector write
-        // outside any Store.transaction()) stays atomic and commits once, not once per row.
+        // Prepares once and loops synchronously, wrapped as one async call so features share one
+        // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.
         async runBatch (sql, paramRows) {
             if (paramRows.length === 0) return;
             await withTransaction(conn, async ()=>{

package/dist/esm/store/sqlite/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every\n// store presents. Every method below does its real work synchronously and returns an already-\n// resolved promise, so reconcile's loop, the feature hooks, and runBatch never actually cross\n// an async boundary internally -- no regression against the pre-Store synchronous code.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously: today's code, unchanged, wrapped as one async call\n    // so features share one Connection contract with DuckDB. Joins the caller's transaction when\n    // there is one (reconcile); otherwise opens its own, so a standalone batch (a vector write\n    // outside any Store.transaction()) stays atomic and commits once, not once per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const stmt = db.prepare(sql);\n        for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,8FAA8F;AAC9F,8FAA8F;AAC9F,wFAAwF;AACxF,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,4FAA4F;QAC5F,6FAA6F;QAC7F,2FAA2F;QAC3F,oFAAoF;QACpF,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBAAgBa,MAAM;gBAC1B,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C;QACF;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real\n// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously, wrapped as one async call so features share one\n    // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const stmt = db.prepare(sql);\n        for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAGpD,wIAAwI;AACxI,6JAA6J;AAC7J,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,yFAAyF;QACzF,4LAA4L;QAC5L,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBAAgBa,MAAM;gBAC1B,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C;QACF;IACF;IACA,OAAOP;AACT"}

package/dist/esm/store/sqlite/fieldStats.js

@@ -1,10 +1,6 @@
 import { quoteIdent } from '../shared.js';
-// One aggregate query, one result row: COUNT + GROUP_CONCAT(DISTINCT typeof()) per column, so
-// the query stays O(columns) rather than O(rows x columns). typeof()'s output already is the
-// shared vocabulary (integer/real/text -- mapValue() never binds a boolean or blob), so no
-// mapping step is needed here (contrast duckdb/fieldStats.ts). GROUP_CONCAT(DISTINCT ...)'s
-// order is not guaranteed sorted (spike-verified: 3+ distinct types come back in insertion
-// order), so the type set is sorted in JS.
+// One aggregate query per column (COUNT + GROUP_CONCAT(DISTINCT typeof())), O(columns) not O(rows x columns);
+// typeof()'s output is already the shared vocabulary, and GROUP_CONCAT's order is unsorted, so JS sorts the type set.
 export async function fieldStats(conn, columns, scopeWhere) {
     const exprs = columns.map((name, i)=>{
         const quoted = quoteIdent(name);

package/dist/esm/store/sqlite/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// One aggregate query, one result row: COUNT + GROUP_CONCAT(DISTINCT typeof()) per column, so\n// the query stays O(columns) rather than O(rows x columns). typeof()'s output already is the\n// shared vocabulary (integer/real/text -- mapValue() never binds a boolean or blob), so no\n// mapping step is needed here (contrast duckdb/fieldStats.ts). GROUP_CONCAT(DISTINCT ...)'s\n// order is not guaranteed sorted (spike-verified: 3+ distinct types come back in insertion\n// order), so the type set is sorted in JS.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => ({\n    field: name,\n    coverage: Number(row[`n${i}`] ?? 0),\n    type: [...new Set(((row[`t${i}`] as string) ?? '').split(',').filter(Boolean))].sort().join(','),\n  }));\n}\n"],"names":["quoteIdent","fieldStats","conn","columns","scopeWhere","exprs","map","name","i","quoted","stmt","prepare","join","row","get","field","coverage","Number","type","Set","split","filter","Boolean","sort"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,8FAA8F;AAC9F,6FAA6F;AAC7F,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,2CAA2C;AAC3C,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACC,MAAMC;QAC/B,MAAMC,SAAST,WAAWO;QAC1B,OAAO,CAAC,MAAM,EAAEE,OAAO,MAAM,EAAED,EAAE,+BAA+B,EAAEC,OAAO,iBAAiB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IAC5H;IACA,MAAME,OAAO,MAAMR,KAAKS,OAAO,CAAC,CAAC,OAAO,EAAEN,MAAMO,IAAI,CAAC,MAAM,kBAAkB,EAAER,YAAY;IAC3F,MAAMS,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOX,QAAQG,GAAG,CAAC,CAACC,MAAMC;YAEPK,OACGA;eAHW;YAC/BE,OAAOR;YACPS,UAAUC,QAAOJ,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAAgB;YACjCK,MAAM;mBAAI,IAAIC,IAAI,EAAEN,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAA2B,IAAIO,KAAK,CAAC,KAAKC,MAAM,CAACC;aAAU,CAACC,IAAI,GAAGX,IAAI,CAAC;QAC9F;;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// One aggregate query per column (COUNT + GROUP_CONCAT(DISTINCT typeof())), O(columns) not O(rows x columns);\n// typeof()'s output is already the shared vocabulary, and GROUP_CONCAT's order is unsorted, so JS sorts the type set.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => ({\n    field: name,\n    coverage: Number(row[`n${i}`] ?? 0),\n    type: [...new Set(((row[`t${i}`] as string) ?? '').split(',').filter(Boolean))].sort().join(','),\n  }));\n}\n"],"names":["quoteIdent","fieldStats","conn","columns","scopeWhere","exprs","map","name","i","quoted","stmt","prepare","join","row","get","field","coverage","Number","type","Set","split","filter","Boolean","sort"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,8GAA8G;AAC9G,sHAAsH;AACtH,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACC,MAAMC;QAC/B,MAAMC,SAAST,WAAWO;QAC1B,OAAO,CAAC,MAAM,EAAEE,OAAO,MAAM,EAAED,EAAE,+BAA+B,EAAEC,OAAO,iBAAiB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IAC5H;IACA,MAAME,OAAO,MAAMR,KAAKS,OAAO,CAAC,CAAC,OAAO,EAAEN,MAAMO,IAAI,CAAC,MAAM,kBAAkB,EAAER,YAAY;IAC3F,MAAMS,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOX,QAAQG,GAAG,CAAC,CAACC,MAAMC;YAEPK,OACGA;eAHW;YAC/BE,OAAOR;YACPS,UAAUC,QAAOJ,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAAgB;YACjCK,MAAM;mBAAI,IAAIC,IAAI,EAAEN,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAA2B,IAAIO,KAAK,CAAC,KAAKC,MAAM,CAACC;aAAU,CAACC,IAAI,GAAGX,IAAI,CAAC;QAC9F;;AACF"}

package/dist/esm/store/sqlite/lexical.js

@@ -1,19 +1,15 @@
 import { segmentMatch } from '../../text/segment.js';
 // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
-// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, hit stays NULL for
-// the caller's JS excerpt fallback.
+// (benchmark/reports/2026-08-23-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.
 const SNIPPET_BOUND = 16384;
 const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
-// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments.
-// `segmenting` mirrors the predicate that decided whether the `_seg` sidecars were populated
-// (contentTokenize(cfg) === undefined, see sqlite/store.ts): an unspaced-script run in `terms`
-// becomes a quoted grapheme phrase against those sidecars only when they exist.
+// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. `segmenting`
+// mirrors the predicate that populated the `_seg` sidecars (contentTokenize(cfg) === undefined): an unspaced-script run in `terms` becomes a quoted grapheme phrase against them only when they exist.
 export async function queryLexical(conn, terms, opts, segmenting) {
     const { whereJoin, whereCond, scopeCond, limit } = opts;
     const query = segmenting ? segmentMatch(terms) : terms;
-    // A run of pure unspaced-script punctuation segments to nothing searchable (segmentField
-    // drops empty groups the same way): an empty MATCH string is an FTS5 syntax error, not zero
-    // rows, so this returns the zero rows directly rather than issuing it.
+    // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH
+    // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.
     if (query.trim() === '') return [];
     const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;
     const stmt = await conn.prepare(sql);

package/dist/esm/store/sqlite/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, hit stays NULL for\n// the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments.\n// `segmenting` mirrors the predicate that decided whether the `_seg` sidecars were populated\n// (contentTokenize(cfg) === undefined, see sqlite/store.ts): an unspaced-script run in `terms`\n// becomes a quoted grapheme phrase against those sidecars only when they exist.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, segmenting: boolean): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmenting ? segmentMatch(terms) : terms;\n  // A run of pure unspaced-script punctuation segments to nothing searchable (segmentField\n  // drops empty groups the same way): an empty MATCH string is an FTS5 syntax error, not zero\n  // rows, so this returns the zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","SNIPPET_BOUND","WEIGHTED_BM25","queryLexical","conn","terms","opts","segmenting","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,iFAAiF;AACjF,6FAA6F;AAC7F,oCAAoC;AACpC,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,uFAAuF;AACvF,6FAA6F;AAC7F,+FAA+F;AAC/F,gFAAgF;AAChF,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB,EAAEC,UAAmB;IAChH,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAMM,QAAQL,aAAaP,aAAaK,SAASA;IACjD,yFAAyF;IACzF,4FAA4F;IAC5F,uEAAuE;IACvE,IAAIO,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,+DAA+D,EAAEb,cAAc,+EAA+E,EAAEO,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAER,cAAc,OAAO,EAAES,OAAO;IACjR,MAAMI,OAAO,MAAMX,KAAKY,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. `segmenting`\n// mirrors the predicate that populated the `_seg` sidecars (contentTokenize(cfg) === undefined): an unspaced-script run in `terms` becomes a quoted grapheme phrase against them only when they exist.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, segmenting: boolean): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmenting ? segmentMatch(terms) : terms;\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","SNIPPET_BOUND","WEIGHTED_BM25","queryLexical","conn","terms","opts","segmenting","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,iFAAiF;AACjF,+HAA+H;AAC/H,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,oGAAoG;AACpG,uMAAuM;AACvM,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB,EAAEC,UAAmB;IAChH,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAMM,QAAQL,aAAaP,aAAaK,SAASA;IACjD,2FAA2F;IAC3F,sGAAsG;IACtG,IAAIO,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,+DAA+D,EAAEb,cAAc,+EAA+E,EAAEO,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAER,cAAc,OAAO,EAAES,OAAO;IACjR,MAAMI,OAAO,MAAMX,KAAKY,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}

package/dist/esm/store/sqlite/open.d.ts

@@ -11,4 +11,3 @@ export interface OpenResult {
 }
 export declare function openSqlite(cfg: ResolvedConfig): Promise<OpenResult>;
 export declare function docCount(store: Store): Promise<number>;
-export declare function clearCache(cfg: ResolvedConfig): void;

package/dist/esm/store/sqlite/open.js

@@ -1,11 +1,13 @@
 // The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with
 // both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.
-import { mkdirSync, rmSync } from 'node:fs';
+import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.js';
+import { rekeyChunkText } from '../../embed/handoff.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
+import { clearCache } from '../cache.js';
 import { getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { createConnection } from './connection.js';
@@ -16,16 +18,11 @@ export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
 export const SCHEMA_VERSION = '18';
-// Stemming is English-only, but the segmentation underneath it is what decides coverage:
-// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)
-// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how
-// such a tree picks trigram instead.
+// unicode61 splits on spaces, so a language written without them indexes a whole run as one
+// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.
 const DEFAULT_TOKENIZE = 'porter unicode61';
-// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a
-// configured value has to be concatenated. Probing a throwaway table is what makes that safe
-// and is also the whole validation: anything the linked SQLite accepts passes, anything else
-// fails here with SQLite's own message rather than against the real table. It means no table
-// of which version added which tokenizer has to be maintained.
+// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured
+// value is concatenated; probing a throwaway table first is both the safety and the validation.
 function resolveTokenize(db, cfg) {
     const configured = contentTokenize(cfg);
     if (configured === undefined) return DEFAULT_TOKENIZE;
@@ -47,11 +44,8 @@ function storedTokenize(db) {
     const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);
     return m ? m[1] : null;
 }
-// The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
-// snippet(content, 2, ...) are documented against the first three columns and keep working
-// (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
-// runs for text that needs it, and an empty string for text that does not. Shared by
-// ensureSchema and the tokenize-only rebuild in connect(), so both create identical DDL.
+// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)
+// are documented against the first three columns. Each holds its field's exploded unspaced runs.
 async function createContentTable(conn, tokenize) {
     await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);
 }
@@ -77,9 +71,8 @@ async function connect(cfg) {
     });
     const dbPath = join(stateDir, DB_FILENAME);
     const db = new DatabaseSync(dbPath);
-    // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,
-    // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.
-    // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.
+    // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and
+    // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.
     let closed = false;
     try {
         var _ref;
@@ -110,16 +103,12 @@ async function connect(cfg) {
                 return connect(cfg);
             }
             const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
-            // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived
-            // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset
-            // edit, an embed model change -- still takes the full clear/reopen below.
+            // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived
+            // and tokenizer-independent. Any other signature change takes the full clear/reopen below.
             if (changedKeys.size === 1 && changedKeys.has('tokenize')) {
                 console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
-                // Drop and recreate as one transaction: a crash before COMMIT rolls back to the table
-                // that was there before (old tokenizer, still queryable), never to no table at all.
-                // IF EXISTS also makes a retry after such a crash a no-op instead of a raw SQLite error --
-                // meta.features is only updated once rebuildContentTable (below) finishes, so a retry
-                // re-enters this branch and repopulates from scratch either way.
+                // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather
+                // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.
                 await withTransaction(conn, async ()=>{
                     await conn.exec('DROP TABLE IF EXISTS content');
                     await createContentTable(conn, tokenize);
@@ -139,10 +128,8 @@ async function connect(cfg) {
                 return connect(cfg);
             }
         }
-        // Meta can lie after a crash between table creation and the signature write; the table's
-        // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild
-        // already recreated content with this tokenize above, so stored already matches and this
-        // guard is skipped naturally rather than needing its own case.
+        // Meta can lie after a crash between table creation and the signature write; the table's own
+        // DDL cannot, so a mismatch here rebuilds whatever meta says.
         const stored = storedTokenize(db);
         if (stored !== null && stored !== tokenize) {
             console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
@@ -182,8 +169,11 @@ async function connect(cfg) {
 // resulting connection in the async Store interface.
 export async function openSqlite(cfg) {
     const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
+    const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);
+    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+    rekeyChunkText(conn, store);
     return {
-        store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir),
+        store: store,
         cfg: resolvedCfg,
         dbPath,
         parsed,
@@ -194,13 +184,3 @@ export async function docCount(store) {
     const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');
     return (await stmt.get()).n;
 }
-// Deletes the cache directory, and only that. The rebuild is not this function's job: the next
-// open() reconciles, which is what running any command already does, so a verb that bundled the
-// two described the half that was not its own. Manual reset for a doubted cache; the schema and
-// config-signature mismatches below reset themselves.
-export function clearCache(cfg) {
-    rmSync(join(cfg.baseDir, STATE_DIR), {
-        recursive: true,
-        force: true
-    });
-}