sensemaking 0.18.2 → 0.18.3

package/dist/esm/store/duckdb/vectors.js

@@ -1,4 +1,5 @@
 import { withTransaction } from '../transaction.js';
+import { asCosine, sampleEvenly } from '../vectors.js';
 import { rewriteUpdate } from './batch.js';
 import { duckdbApi } from './native.js';
 // This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by
@@ -16,16 +17,6 @@ import { duckdbApi } from './native.js';
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 const untyped = undefined;
-// Seed chunks that participate in a `related` scan; must match sqlite/vectors.ts's
-// TARGET_CHUNK_CAP for identical sampling behavior (see that file for the measurement it's
-// based on -- 12.7s at 201 chunks/note without sampling).
-const TARGET_CHUNK_CAP = 16;
-// array_cosine_similarity is already a true cosine (no int8 accumulation to push slightly
-// outside [-1, 1]), but rounded the same way sqlite's scores are so both stores print the
-// same bounded number rather than one having more decimal noise than the other.
-function asCosine(score) {
-    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
-}
 // The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced
 // under the column's width); zero-padding leaves cosine scores unchanged since the added
 // dimensions contribute nothing to either vector's dot product or norm.
@@ -124,8 +115,7 @@ export async function scanSimilar(duckdb, conn, dims, path, opts) {
     const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');
     const targetRows = await targetStmt.all(path);
     if (targetRows.length === 0) return [];
-    const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
-    const targets = targetRows.filter((_, i)=>i % step === 0).map((row)=>row.vector);
+    const targets = sampleEvenly(targetRows).map((row)=>row.vector);
     const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
     const excludeList = [
         ...opts.exclude

package/dist/esm/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by\n// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with\n// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --\n// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own\n// representation and stays as it is. Every function here takes `dims` as a parameter rather\n// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real\n// constant at the call site).\n//\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only\n// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's\n// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever\n// open.ts already resolved, including via an on-demand install (see native.ts).\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// Seed chunks that participate in a `related` scan; must match sqlite/vectors.ts's\n// TARGET_CHUNK_CAP for identical sampling behavior (see that file for the measurement it's\n// based on -- 12.7s at 201 chunks/note without sampling).\nconst TARGET_CHUNK_CAP = 16;\n\n// array_cosine_similarity is already a true cosine (no int8 accumulation to push slightly\n// outside [-1, 1]), but rounded the same way sqlite's scores are so both stores print the\n// same bounded number rather than one having more decimal noise than the other.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced\n// under the column's width); zero-padding leaves cosine scores unchanged since the added\n// dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces\n// for every store, see toStore); dequantizing into a plain float array is this store's own\n// representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's\n// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native\n// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --\n// automatic type inference on the raw values misreads an all-integer first component (a real\n// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row\n// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks\n// the winning chunk's line range from the same row max(score) came from, per path. An empty\n// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed\n// into one native scan: the sampled target vectors become a small VALUES list, cross-joined\n// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost\n// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const targets = targetRows.filter((_, i) => i % step === 0).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","rewriteUpdate","duckdbApi","untyped","undefined","TARGET_CHUNK_CAP","asCosine","score","Math","round","min","max","padded","values","dims","out","Array","fill","d","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","scanSimilar","opts","targetStmt","targetRows","all","step","ceil","targets","filter","_","i","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,mFAAmF;AACnF,uFAAuF;AACvF,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAC5F,8FAA8F;AAC9F,8BAA8B;AAC9B,EAAE;AACF,4FAA4F;AAC5F,8FAA8F;AAC9F,gGAAgG;AAChG,gFAAgF;AAEhF,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,mFAAmF;AACnF,2FAA2F;AAC3F,0DAA0D;AAC1D,MAAMC,mBAAmB;AAEzB,0FAA0F;AAC1F,0FAA0F;AAC1F,gFAAgF;AAChF,SAASC,SAASC,KAAa;IAC7B,OAAOC,KAAKC,KAAK,CAACD,KAAKE,GAAG,CAAC,GAAGF,KAAKG,GAAG,CAAC,CAAC,GAAGJ,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,wEAAwE;AACxE,SAASK,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIV,KAAKE,GAAG,CAACG,OAAOM,MAAM,EAAEL,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASK,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEL,MAAMO,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,2FAA2F;AAC3F,SAASC,WAAWC,GAAmB,EAAEZ,IAAY;IACnD,MAAMa,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOpB,OACLI,MAAMO,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCpB;AAEJ;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,0FAA0F;AAC1F,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,eAAeqB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEvB,IAAY,EAAEwB,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMvC;IAC5C,MAAMwC,YAAYzC,cAAc,mEAAmEqC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAM3C,gBAAgBqC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMjC,SAAwB,EAAE;YAChC,MAAMkC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtBzB,OAAOmC,IAAI,CAACT,WAAWd,WAAWC,KAAKZ,QAAQY,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ3B,OAAOX,SAASA;YAC3C;YACAyC,KAAKO,IAAI,CAACtC,QAAQkC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,6FAA6F;AAC7F,4FAA4F;AAC5F,2FAA2F;AAC3F,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAEzC,IAAY,EAAE0C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMvC;IAC5C,MAAMyD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMjC,SAAwB;YAAC0B,WAAW3B,OAAO2C,IAAIzC;eAAW6C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ3B;eAAU6C,YAAYC,GAAG,CAAC,IAAMzD;YAAUA;SAAQ;QAC7FyC,KAAKO,IAAI,CAACtC,QAAQkC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAY9D,SAAS0D,EAAEzD,KAAK;YAAE,CAAA;IACjH,SAAU;QACRqC,KAAKS,WAAW;IAClB;AACF;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,8FAA8F;AAC9F,wFAAwF;AACxF,OAAO,eAAegB,YAAYjC,MAAwB,EAAEC,IAAgB,EAAEvB,IAAY,EAAEmC,IAAY,EAAEqB,IAAgE;IACxK,IAAIA,KAAKb,OAAO,IAAIa,KAAKb,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMa,aAAa,MAAMlC,KAAKQ,OAAO,CAAC;IACtC,MAAM2B,aAAc,MAAMD,WAAWE,GAAG,CAACxB;IACzC,IAAIuB,WAAWrD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMuD,OAAOlE,KAAKG,GAAG,CAAC,GAAGH,KAAKmE,IAAI,CAACH,WAAWrD,MAAM,GAAGd;IACvD,MAAMuE,UAAUJ,WAAWK,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIL,SAAS,GAAGd,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEnF,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMvC;IAC5C,MAAM8E,cAAc;WAAIV,KAAKW,OAAO;KAAC;IACrC,MAAMtB,cAAcW,KAAKb,OAAO,GAAG;WAAIa,KAAKb,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE8B,QAAQhB,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEwD,YAAY7D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAY4D,YAAY7D,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEmD,KAAKb,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMjC,SAAwB;eAAI+D,QAAQhB,GAAG,CAAC,CAACsB,IAAM3C,WAAW3B,OAAOsE,GAAGpE;YAASmC;eAAS+B;eAAgBrB;YAAaW,KAAKa,CAAC;SAAC;QAChI,MAAMpC,QAAsB;eAAI6B,QAAQhB,GAAG,CAAC,IAAMpB,MAAMC,QAAQ3B;YAAQX;eAAY6E,YAAYpB,GAAG,CAAC,IAAMzD;eAAawD,YAAYC,GAAG,CAAC,IAAMzD;YAAUA;SAAQ;QAC/JyC,KAAKO,IAAI,CAACtC,QAAQkC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAY9D,SAAS0D,EAAEzD,KAAK;YAAE,CAAA;IACxE,SAAU;QACRqC,KAAKS,WAAW;IAClB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by\n// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with\n// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --\n// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own\n// representation and stays as it is. Every function here takes `dims` as a parameter rather\n// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real\n// constant at the call site).\n//\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only\n// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's\n// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever\n// open.ts already resolved, including via an on-demand install (see native.ts).\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced\n// under the column's width); zero-padding leaves cosine scores unchanged since the added\n// dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces\n// for every store, see toStore); dequantizing into a plain float array is this store's own\n// representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's\n// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native\n// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --\n// automatic type inference on the raw values misreads an all-integer first component (a real\n// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row\n// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks\n// the winning chunk's line range from the same row max(score) came from, per path. An empty\n// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed\n// into one native scan: the sampled target vectors become a small VALUES list, cross-joined\n// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost\n// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","score","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,mFAAmF;AACnF,uFAAuF;AACvF,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAC5F,8FAA8F;AAC9F,8BAA8B;AAC9B,EAAE;AACF,4FAA4F;AAC5F,8FAA8F;AAC9F,gGAAgG;AAChG,gFAAgF;AAEhF,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,6FAA6F;AAC7F,yFAAyF;AACzF,wEAAwE;AACxE,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,2FAA2F;AAC3F,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,0FAA0F;AAC1F,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,6FAA6F;AAC7F,4FAA4F;AAC5F,2FAA2F;AAC3F,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;YAAC4B,WAAW7B,OAAO6C,IAAI3C;eAAW+C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ7B;eAAU+C,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC7FoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACjH,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,8FAA8F;AAC9F,wFAAwF;AACxF,OAAO,eAAeiB,YAAYlC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAEsB,IAAgE;IACxK,IAAIA,KAAKd,OAAO,IAAIc,KAAKd,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMc,aAAa,MAAMnC,KAAKQ,OAAO,CAAC;IACtC,MAAM4B,aAAc,MAAMD,WAAWE,GAAG,CAACzB;IACzC,IAAIwB,WAAWtD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUtE,aAAaoE,YAAYb,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMqE,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMlB,cAAcY,KAAKd,OAAO,GAAG;WAAIc,KAAKd,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE6B,QAAQf,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEoD,YAAYzD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYwD,YAAYzD,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEoD,KAAKd,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIgE,QAAQf,GAAG,CAAC,CAACkB,IAAMvC,WAAW7B,OAAOoE,GAAGlE;YAASqC;eAAS2B;eAAgBjB;YAAaY,KAAKQ,CAAC;SAAC;QAChI,MAAMhC,QAAsB;eAAI4B,QAAQf,GAAG,CAAC,IAAMpB,MAAMC,QAAQ7B;YAAQJ;eAAYoE,YAAYhB,GAAG,CAAC,IAAMpD;eAAamD,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACxE,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF"}

package/dist/esm/store/index.js

@@ -6,6 +6,8 @@ import { openDuckdb } from './duckdb/open.js';
 import { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.js';
 import { openSqlite } from './sqlite/open.js';
 import { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.js';
+import { openTurso } from './turso/open.js';
+import { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.js';
 // Capabilities are checked against the registry entry before opening (no wasted connect/reconcile
 // work for a config that cannot be satisfied), then the store itself opens.
 const REGISTRY = {
@@ -16,6 +18,10 @@ const REGISTRY = {
     duckdb: {
         capabilities: DUCKDB_CAPABILITIES,
         open: openDuckdb
+    },
+    turso: {
+        capabilities: TURSO_CAPABILITIES,
+        open: openTurso
     }
 };
 function entryFor(name) {

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Capability } from './types.ts';\n\ntype StoreName = 'sqlite' | 'duckdb';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually\n  // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level\n  // (every preset defaults to wanting \"words\"), so a store lacking 'lexical' fails loudly at\n  // first use instead (see duckdb/store.ts's lexical.query()).\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { getMeta, setMeta } from './meta.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","getMeta","setMeta","clearCache","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AAUxE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcH;QAAqBI,MAAML;IAAW;IAC9DM,QAAQ;QAAEF,cAAcL;QAAqBM,MAAMR;IAAW;AAChE;AAEA,SAASU,SAASC,IAAe;IAC/B,MAAMC,QAAQP,QAAQ,CAACM,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIb,WAAW,iBAAiB,CAAC,uBAAuB,EAAEY,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACT,UAAUU,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOb,UAAUmB;IACvB,MAAML,QAAQF,SAASC;IAEvB,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,6DAA6D;IAC7D,IAAId,gBAAgBoB,QAAQ,CAACL,MAAML,YAAY,CAACW,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAInB,WAAW,4BAA4B,CAAC,OAAO,EAAEY,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMJ,IAAI,CAACS;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOb,UAAUmB;IACvB,IAAI,CAACP,SAASC,MAAMJ,YAAY,CAACW,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAInB,WAAW,4BAA4B,CAAC,OAAO,EAAEY,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,OAAO,EAAEC,OAAO,QAAQ,YAAY;AAE7C,SAASC,UAAU,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually\n  // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level\n  // (every preset defaults to wanting \"words\"), so a store lacking 'lexical' fails loudly at\n  // first use instead (see duckdb/store.ts's lexical.query()).\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { getMeta, setMeta } from './meta.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","getMeta","setMeta","clearCache","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,6DAA6D;IAC7D,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,OAAO,EAAEC,OAAO,QAAQ,YAAY;AAE7C,SAASC,UAAU,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}

package/dist/esm/store/native.d.ts

@@ -0,0 +1,7 @@
+export interface NativeDescriptor {
+    store: string;
+    pkg: string;
+    sizeHint: string;
+}
+export declare function packageNodeModules(): string;
+export declare function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName?: string): Promise<T>;

package/dist/esm/store/native.js

@@ -0,0 +1,83 @@
+import { existsSync, readFileSync } from 'node:fs';
+import Module from 'node:module';
+import { dirname, join } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { SenseError } from '../errors.js';
+// install-module-linked itself is small, but a target native package can be large (@duckdb/node-
+// api's ~110MB is the current instance): deferred the same way embed/static.ts defers
+// @huggingface/tokenizers, so requiring it stays inside the already-gated branch below rather
+// than running for every store/index.ts load.
+const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
+const _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));
+// Walks up from this file's own directory to find the real package.json, rather than a fixed
+// dot-count from __dirname (xz-compat's `../../node_modules`): this file sits at a different
+// depth under src/ during `tsds test:node` than under the built dist/{cjs,esm}/store output, so a
+// hardcoded depth is right for one and wrong for the other. Works the same way for a
+// globally-installed `sense`, whose node_modules this resolves to -- never the caller's cwd.
+export function packageNodeModules() {
+    let dir = _dirname;
+    for(;;){
+        const candidate = join(dir, 'package.json');
+        if (existsSync(candidate)) {
+            const pkg = JSON.parse(readFileSync(candidate, 'utf8'));
+            if (pkg.name === 'sensemaking') return join(dir, 'node_modules');
+        }
+        const parent = dirname(dir);
+        if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');
+        dir = parent;
+    }
+}
+const installing = new Map();
+// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).
+// The install here is async -- the load site already awaits a dynamic import -- so a bare
+// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target
+// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,
+// keyed by specifier and target so tests exercising several installs in one process don't share
+// one slot. A failed install is not cached, so a later call retries.
+function installOnce(specifier, nodeModulesPath) {
+    const key = `${specifier}\n${nodeModulesPath}`;
+    let promise = installing.get(key);
+    if (!promise) {
+        const installModuleLinked = _require('install-module-linked').default;
+        promise = installModuleLinked(specifier, nodeModulesPath).then(()=>{}, (err)=>{
+            installing.delete(key);
+            throw err;
+        });
+        installing.set(key, promise);
+    }
+    return promise;
+}
+// Node caches a bare specifier's package resolution -- including a "not found" miss -- for the
+// life of the process, so retrying `import(importName)` right after installing sees the same
+// stale miss and falls through to a bogus index.js guess instead of the package's real entry
+// (verified against a real global install: the plain-specifier retry failed there, this did
+// not). Reading the freshly-installed package's own package.json and importing its entry file
+// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.
+function resolveEntryUrl(importName, nodeModulesPath) {
+    var _pkg_main;
+    const pkgDir = join(nodeModulesPath, ...importName.split('/'));
+    const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8'));
+    const entry = typeof pkg.exports === 'string' ? pkg.exports : (_pkg_main = pkg.main) !== null && _pkg_main !== void 0 ? _pkg_main : 'index.js';
+    return pathToFileURL(join(pkgDir, entry)).href;
+}
+// Import-then-install-then-retry, over an injectable importName so the failure path (import AND
+// install both fail) can be exercised for real in tests without downloading a real store's native
+// binding. importName defaults to descriptor.pkg, which is what production always wants; the
+// thrown errors name descriptor.pkg regardless, since that is what a real user needs to install.
+export async function loadOrInstall(descriptor, nodeModulesPath, importName = descriptor.pkg) {
+    try {
+        return await import(importName);
+    } catch  {
+        console.error(`sense: store "${descriptor.store}" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);
+        try {
+            await installOnce(importName, nodeModulesPath);
+        } catch (installErr) {
+            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" needs ${descriptor.pkg}, and installing it automatically failed (${installErr.message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \`npm install ${descriptor.pkg}\` yourself and try again`);
+        }
+        try {
+            return await import(resolveEntryUrl(importName, nodeModulesPath));
+        } catch (loadErr) {
+            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "${descriptor.store}" installed ${descriptor.pkg} but it could not be loaded (${loadErr.message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \`npm install ${descriptor.pkg}\` to see the underlying error`);
+        }
+    }
+}

package/dist/esm/store/native.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked itself is small, but a target native package can be large (@duckdb/node-\n// api's ~110MB is the current instance): deferred the same way embed/static.ts defers\n// @huggingface/tokenizers, so requiring it stays inside the already-gated branch below rather\n// than running for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory to find the real package.json, rather than a fixed\n// dot-count from __dirname (xz-compat's `../../node_modules`): this file sits at a different\n// depth under src/ during `tsds test:node` than under the built dist/{cjs,esm}/store output, so a\n// hardcoded depth is right for one and wrong for the other. Works the same way for a\n// globally-installed `sense`, whose node_modules this resolves to -- never the caller's cwd.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).\n// The install here is async -- the load site already awaits a dynamic import -- so a bare\n// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target\n// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,\n// keyed by specifier and target so tests exercising several installs in one process don't share\n// one slot. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's package resolution -- including a \"not found\" miss -- for the\n// life of the process, so retrying `import(importName)` right after installing sees the same\n// stale miss and falls through to a bogus index.js guess instead of the package's real entry\n// (verified against a real global install: the plain-specifier retry failed there, this did\n// not). Reading the freshly-installed package's own package.json and importing its entry file\n// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path (import AND\n// install both fail) can be exercised for real in tests without downloading a real store's native\n// binding. importName defaults to descriptor.pkg, which is what production always wants; the\n// thrown errors name descriptor.pkg regardless, since that is what a real user needs to install.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,iGAAiG;AACjG,sFAAsF;AACtF,8FAA8F;AAC9F,8CAA8C;AAC9C,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,6FAA6F;AAC7F,6FAA6F;AAC7F,kGAAkG;AAClG,qFAAqF;AACrF,6FAA6F;AAC7F,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,qEAAqE;AACrE,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,6FAA6F;AAC7F,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOX,cAAcF,KAAKmC,QAAQE,QAAQG,IAAI;AAChD;AAEA,gGAAgG;AAChG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,OAAO,eAAeC,cAAiBC,UAA4B,EAAEnB,eAAuB,EAAEW,aAAqBQ,WAAW7B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNS,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,wCAAwC,EAAE6B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAMzB,YAAYa,YAAYX;QAChC,EAAE,OAAOwB,YAAY;YACnB,MAAM,IAAI5C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,0CAA0C,EAAE,AAACkC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW7B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,gBAAgBC,YAAYX;QACnD,EAAE,OAAO0B,SAAS;YAChB,MAAM,IAAI9C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW7B,GAAG,CAAC,6BAA6B,EAAE,AAACoC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW7B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}

package/dist/esm/store/sqlite/fieldStats.d.ts

@@ -0,0 +1,2 @@
+import type { Connection, FieldStat } from '../types.js';
+export declare function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]>;

package/dist/esm/store/sqlite/fieldStats.js

@@ -0,0 +1,25 @@
+import { quoteIdent } from '../shared.js';
+// One aggregate query, one result row: COUNT + GROUP_CONCAT(DISTINCT typeof()) per column, so
+// the query stays O(columns) rather than O(rows x columns). typeof()'s output already is the
+// shared vocabulary (integer/real/text -- mapValue() never binds a boolean or blob), so no
+// mapping step is needed here (contrast duckdb/fieldStats.ts). GROUP_CONCAT(DISTINCT ...)'s
+// order is not guaranteed sorted (spike-verified: 3+ distinct types come back in insertion
+// order), so the type set is sorted in JS.
+export async function fieldStats(conn, columns, scopeWhere) {
+    const exprs = columns.map((name, i)=>{
+        const quoted = quoteIdent(name);
+        return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;
+    });
+    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);
+    const row = await stmt.get();
+    return columns.map((name, i)=>{
+        var _row_, _row_1;
+        return {
+            field: name,
+            coverage: Number((_row_ = row[`n${i}`]) !== null && _row_ !== void 0 ? _row_ : 0),
+            type: [
+                ...new Set(((_row_1 = row[`t${i}`]) !== null && _row_1 !== void 0 ? _row_1 : '').split(',').filter(Boolean))
+            ].sort().join(',')
+        };
+    });
+}

package/dist/esm/store/sqlite/fieldStats.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// One aggregate query, one result row: COUNT + GROUP_CONCAT(DISTINCT typeof()) per column, so\n// the query stays O(columns) rather than O(rows x columns). typeof()'s output already is the\n// shared vocabulary (integer/real/text -- mapValue() never binds a boolean or blob), so no\n// mapping step is needed here (contrast duckdb/fieldStats.ts). GROUP_CONCAT(DISTINCT ...)'s\n// order is not guaranteed sorted (spike-verified: 3+ distinct types come back in insertion\n// order), so the type set is sorted in JS.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => ({\n    field: name,\n    coverage: Number(row[`n${i}`] ?? 0),\n    type: [...new Set(((row[`t${i}`] as string) ?? '').split(',').filter(Boolean))].sort().join(','),\n  }));\n}\n"],"names":["quoteIdent","fieldStats","conn","columns","scopeWhere","exprs","map","name","i","quoted","stmt","prepare","join","row","get","field","coverage","Number","type","Set","split","filter","Boolean","sort"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,8FAA8F;AAC9F,6FAA6F;AAC7F,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,2CAA2C;AAC3C,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACC,MAAMC;QAC/B,MAAMC,SAAST,WAAWO;QAC1B,OAAO,CAAC,MAAM,EAAEE,OAAO,MAAM,EAAED,EAAE,+BAA+B,EAAEC,OAAO,iBAAiB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IAC5H;IACA,MAAME,OAAO,MAAMR,KAAKS,OAAO,CAAC,CAAC,OAAO,EAAEN,MAAMO,IAAI,CAAC,MAAM,kBAAkB,EAAER,YAAY;IAC3F,MAAMS,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOX,QAAQG,GAAG,CAAC,CAACC,MAAMC;YAEPK,OACGA;eAHW;YAC/BE,OAAOR;YACPS,UAAUC,QAAOJ,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAAgB;YACjCK,MAAM;mBAAI,IAAIC,IAAI,EAAEN,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAA2B,IAAIO,KAAK,CAAC,KAAKC,MAAM,CAACC;aAAU,CAACC,IAAI,GAAGX,IAAI,CAAC;QAC9F;;AACF"}

package/dist/esm/store/sqlite/open.js

@@ -7,6 +7,7 @@ import { contentTokenize, featureSignature, STATE_DIR } from '../../config/index
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
 import { getMeta, setMeta } from '../shared.js';
+import { withTransaction } from '../transaction.js';
 import { createConnection } from './connection.js';
 import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
@@ -46,17 +47,21 @@ function storedTokenize(db) {
     const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);
     return m ? m[1] : null;
 }
+// The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
+// snippet(content, 2, ...) are documented against the first three columns and keep working
+// (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
+// runs for text that needs it, and an empty string for text that does not. Shared by
+// ensureSchema and the tokenize-only rebuild in connect(), so both create identical DDL.
+async function createContentTable(conn, tokenize) {
+    await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);
+}
 // Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
 // can't dump file text into context. Features add their own tables after the core ones.
 async function ensureSchema(conn, cfg, tokenize) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
     // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL
     // against the resolved tokenizer before this runs, so a stale table is already gone by now.
-    // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
-    // snippet(content, 2, ...) are documented against the first three columns and keep working
-    // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
-    // runs for text that needs it, and an empty string for text that does not.
-    await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);
+    await createContentTable(conn, tokenize);
     // Coverage, not ownership: a path can appear under several presets. path leads the PK so the
     // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
@@ -110,7 +115,15 @@ async function connect(cfg) {
             // edit, an embed model change -- still takes the full clear/reopen below.
             if (changedKeys.size === 1 && changedKeys.has('tokenize')) {
                 console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
-                db.exec('DROP TABLE content');
+                // Drop and recreate as one transaction: a crash before COMMIT rolls back to the table
+                // that was there before (old tokenizer, still queryable), never to no table at all.
+                // IF EXISTS also makes a retry after such a crash a no-op instead of a raw SQLite error --
+                // meta.features is only updated once rebuildContentTable (below) finishes, so a retry
+                // re-enters this branch and repopulates from scratch either way.
+                await withTransaction(conn, async ()=>{
+                    await conn.exec('DROP TABLE IF EXISTS content');
+                    await createContentTable(conn, tokenize);
+                });
                 tokenizeOnlyRebuild = true;
             } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
                 // First sight of a resolved weight identity: the model itself hasn't changed, so
@@ -128,8 +141,8 @@ async function connect(cfg) {
         }
         // Meta can lie after a crash between table creation and the signature write; the table's
         // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild
-        // just dropped content above, so storedTokenize sees no table and this guard is skipped
-        // naturally rather than needing its own case.
+        // already recreated content with this tokenize above, so stored already matches and this
+        // guard is skipped naturally rather than needing its own case.
         const stored = storedTokenize(db);
         if (stored !== null && stored !== tokenize) {
             console.error('sense: cache was built with a different content tokenizer; rebuilding the index');

package/dist/esm/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n  // snippet(content, 2, ...) are documented against the first three columns and keep working\n  // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n  // runs for text that needs it, and an empty string for text that does not.\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,\n  // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.\n  // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.\n  let closed = false;\n  try {\n    db.exec('PRAGMA journal_mode = WAL');\n    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n    // that outwaits it still fails loudly.\n    db.exec('PRAGMA busy_timeout = 30000');\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n      // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n      // edit, an embed model change -- still takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        db.exec('DROP TABLE content');\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's\n    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n    // just dropped content above, so storedTokenize sees no table and this guard is skipped\n    // naturally rather than needing its own case.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg, tokenize);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","SenseError","activeFeatures","FEATURES","getMeta","setMeta","createConnection","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","ensureSchema","conn","tokenize","feature","schema","connect","stateDir","baseDir","recursive","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","clearCache","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","openSqlite","resolvedCfg","store","docCount","stmt","n","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACrF,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,aAAa,QAAQ,iBAAiB;AAC3H,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAmBnC,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,MAAMC,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAatB,gBAAgBqB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIxB,WAAW,kBAAkB,CAAC,kBAAkB,EAAEmB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeG,aAAaC,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IACzE,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,4FAA4F;IAC5F,2FAA2F;IAC3F,2FAA2F;IAC3F,2EAA2E;IAC3E,MAAMW,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;IACzK,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMa,WAAWnC,eAAeiB,KAAM,MAAMkB,QAAQC,MAAM,CAACH;IAChE,IAAI,AAAC,MAAM/B,QAAQ+B,MAAM,sBAAuB,MAAM,MAAM9B,QAAQ8B,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMX,QAAQ+B,MAAM,gBAAiB,MAAM,MAAM9B,QAAQ8B,MAAM,YAAYpC,iBAAiBoB,KAAKhB;AACxG;AAEA,eAAeoC,QAAQpB,GAAmB;IACxC,MAAMqB,WAAW5C,KAAKuB,IAAIsB,OAAO,EAAEzC;IACnCN,UAAU8C,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS/C,KAAK4C,UAAU1B;IAE9B,MAAMI,KAAK,IAAIrB,aAAa8C;IAC5B,8FAA8F;IAC9F,6FAA6F;IAC7F,8FAA8F;IAC9F,IAAIC,SAAS;IACb,IAAI;YA4E4B;QA3E9B1B,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,uCAAuC;QACvCN,GAAGM,IAAI,CAAC;QACRZ,kBAAkBM,IAAIpB,gBAAgBqB,SAASE;QAC/C,MAAMc,OAAO7B,iBAAiBY;QAE9BA,GAAGM,IAAI,CAAC;QAER,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMY,WAAWnB,gBAAgBC,IAAIC;QAErC,uFAAuF;QACvF,qGAAqG;QACrG,MAAM0B,UAAU,MAAMzC,QAAQ+B,MAAM;QACpC,MAAMW,WAAW,MAAM1C,QAAQ+B,MAAM;QACrC,MAAMY,eAAehD,iBAAiBoB,KAAKhB;QAC3C,IAAI6C,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAY9B,kBAAoB+B,aAAa,QAAQA,aAAaC,cAAe;YACxG,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAY9B,gBAAgB;gBAClDkC,QAAQC,KAAK,CAAC;gBACdN,SAAS;gBACT1B,GAAGiC,KAAK;gBACRC,WAAWjC;gBACX,OAAOoB,QAAQpB;YACjB;YACA,MAAMkC,cAAc9C,qBAAqBuC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,0FAA0F;YAC1F,yFAAyF;YACzF,0EAA0E;YAC1E,IAAIM,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;gBACzDN,QAAQC,KAAK,CAAC;gBACdhC,GAAGM,IAAI,CAAC;gBACRwB,sBAAsB;YACxB,OAAO,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAY/C,qBAAqBsC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFE,QAAQC,KAAK,CAAC;gBACd,MAAM7C,QAAQ8B,MAAM,YAAYY;YAClC,OAAO;gBACL,MAAMS,UAAU7C,cAAcmC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEM,QAAQ,oBAAoB,CAAC;gBACpEZ,SAAS;gBACT1B,GAAGiC,KAAK;gBACRC,WAAWjC;gBACX,OAAOoB,QAAQpB;YACjB;QACF;QAEA,yFAAyF;QACzF,6FAA6F;QAC7F,wFAAwF;QACxF,8CAA8C;QAC9C,MAAMsC,SAAS9B,eAAeT;QAC9B,IAAIuC,WAAW,QAAQA,WAAWrB,UAAU;YAC1Ca,QAAQC,KAAK,CAAC;YACdN,SAAS;YACT1B,GAAGiC,KAAK;YACRC,WAAWjC;YACX,OAAOoB,QAAQpB;QACjB;QAEA,MAAMe,aAAaC,MAAMhB,KAAKiB;QAE9B,IAAIsB,kBAA4B,EAAE;QAClC,IAAIV,qBAAqB;YACvBU,kBAAkB,MAAMjD,oBAAoB0B,MAAMhB,KAAKA,IAAIsB,OAAO;YAClE,MAAMpC,QAAQ8B,MAAM,YAAYY;QAClC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMY,gBAAgBC,QAAQ,OAAA,MAAMxD,QAAQ+B,MAAM,iCAApB,kBAAA,OAA4C;QAC1EjB,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAEqC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMvD,UAAUyB,MAAMhB,KAAKA,IAAIsB,OAAO;QAEnE,OAAO;YAAEvB;YAAIiB;YAAMhB;YAAKwB;YAAQqB;YAAQC,UAAU;mBAAIP;mBAAoBO;aAAS;QAAC;IACtF,EAAE,OAAOxC,KAAK;QACZ,IAAI,CAACmB,QAAQ1B,GAAGiC,KAAK;QACrB,MAAM1B;IACR;AACF;AAEA,wFAAwF;AACxF,qDAAqD;AACrD,OAAO,eAAeyC,WAAW/C,GAAmB;IAClD,MAAM,EAAED,EAAE,EAAEiB,IAAI,EAAEhB,KAAKgD,WAAW,EAAExB,MAAM,EAAEqB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAM1B,QAAQpB;IAC/E,OAAO;QAAEiD,OAAOvD,YAAYK,IAAIiB,MAAMgC,aAAaA,YAAY1B,OAAO;QAAGtB,KAAKgD;QAAaxB;QAAQqB;QAAQC;IAAS;AACtH;AAEA,OAAO,eAAeI,SAASD,KAAY;IACzC,MAAME,OAAO,MAAMF,MAAMvC,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMyC,KAAKxC,GAAG,EAAC,EAAqByC,CAAC;AAChD;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAASnB,WAAWjC,GAAmB;IAC5CxB,OAAOC,KAAKuB,IAAIsB,OAAO,EAAEzC,YAAY;QAAE0C,WAAW;QAAM8B,OAAO;IAAK;AACtE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n// snippet(content, 2, ...) are documented against the first three columns and keep working\n// (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n// runs for text that needs it, and an empty string for text that does not. Shared by\n// ensureSchema and the tokenize-only rebuild in connect(), so both create identical DDL.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,\n  // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.\n  // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.\n  let closed = false;\n  try {\n    db.exec('PRAGMA journal_mode = WAL');\n    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n    // that outwaits it still fails loudly.\n    db.exec('PRAGMA busy_timeout = 30000');\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n      // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n      // edit, an embed model change -- still takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // Drop and recreate as one transaction: a crash before COMMIT rolls back to the table\n        // that was there before (old tokenizer, still queryable), never to no table at all.\n        // IF EXISTS also makes a retry after such a crash a no-op instead of a raw SQLite error --\n        // meta.features is only updated once rebuildContentTable (below) finishes, so a retry\n        // re-enters this branch and repopulates from scratch either way.\n        await withTransaction(conn, async () => {\n          await conn.exec('DROP TABLE IF EXISTS content');\n          await createContentTable(conn, tokenize);\n        });\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's\n    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n    // already recreated content with this tokenize above, so stored already matches and this\n    // guard is skipped naturally rather than needing its own case.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg, tokenize);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","SenseError","activeFeatures","FEATURES","getMeta","setMeta","withTransaction","createConnection","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","schema","connect","stateDir","baseDir","recursive","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","clearCache","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","openSqlite","resolvedCfg","store","docCount","stmt","n","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACrF,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,aAAa,QAAQ,iBAAiB;AAC3H,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAmBnC,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,MAAMC,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAavB,gBAAgBsB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIzB,WAAW,kBAAkB,CAAC,kBAAkB,EAAEoB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,4FAA4F;AAC5F,2FAA2F;AAC3F,2FAA2F;AAC3F,qFAAqF;AACrF,yFAAyF;AACzF,eAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;IAClE,MAAMD,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeC,aAAaF,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IACzE,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,MAAMU,mBAAmBC,MAAMC;IAC/B,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMc,WAAWrC,eAAekB,KAAM,MAAMmB,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMhC,QAAQgC,MAAM,sBAAuB,MAAM,MAAM/B,QAAQ+B,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMZ,QAAQgC,MAAM,gBAAiB,MAAM,MAAM/B,QAAQ+B,MAAM,YAAYrC,iBAAiBqB,KAAKjB;AACxG;AAEA,eAAesC,QAAQrB,GAAmB;IACxC,MAAMsB,WAAW9C,KAAKwB,IAAIuB,OAAO,EAAE3C;IACnCN,UAAUgD,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAASjD,KAAK8C,UAAU3B;IAE9B,MAAMI,KAAK,IAAItB,aAAagD;IAC5B,8FAA8F;IAC9F,6FAA6F;IAC7F,8FAA8F;IAC9F,IAAIC,SAAS;IACb,IAAI;YAoF4B;QAnF9B3B,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,uCAAuC;QACvCN,GAAGM,IAAI,CAAC;QACRZ,kBAAkBM,IAAIrB,gBAAgBsB,SAASE;QAC/C,MAAMc,OAAO7B,iBAAiBY;QAE9BA,GAAGM,IAAI,CAAC;QAER,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMY,WAAWnB,gBAAgBC,IAAIC;QAErC,uFAAuF;QACvF,qGAAqG;QACrG,MAAM2B,UAAU,MAAM3C,QAAQgC,MAAM;QACpC,MAAMY,WAAW,MAAM5C,QAAQgC,MAAM;QACrC,MAAMa,eAAelD,iBAAiBqB,KAAKjB;QAC3C,IAAI+C,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAY/B,kBAAoBgC,aAAa,QAAQA,aAAaC,cAAe;YACxG,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAY/B,gBAAgB;gBAClDmC,QAAQC,KAAK,CAAC;gBACdN,SAAS;gBACT3B,GAAGkC,KAAK;gBACRC,WAAWlC;gBACX,OAAOqB,QAAQrB;YACjB;YACA,MAAMmC,cAAc/C,qBAAqBwC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,0FAA0F;YAC1F,yFAAyF;YACzF,0EAA0E;YAC1E,IAAIM,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;gBACzDN,QAAQC,KAAK,CAAC;gBACd,sFAAsF;gBACtF,oFAAoF;gBACpF,2FAA2F;gBAC3F,sFAAsF;gBACtF,iEAAiE;gBACjE,MAAM9C,gBAAgB8B,MAAM;oBAC1B,MAAMA,KAAKX,IAAI,CAAC;oBAChB,MAAMU,mBAAmBC,MAAMC;gBACjC;gBACAa,sBAAsB;YACxB,OAAO,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAYhD,qBAAqBuC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFE,QAAQC,KAAK,CAAC;gBACd,MAAM/C,QAAQ+B,MAAM,YAAYa;YAClC,OAAO;gBACL,MAAMS,UAAU9C,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEM,QAAQ,oBAAoB,CAAC;gBACpEZ,SAAS;gBACT3B,GAAGkC,KAAK;gBACRC,WAAWlC;gBACX,OAAOqB,QAAQrB;YACjB;QACF;QAEA,yFAAyF;QACzF,6FAA6F;QAC7F,yFAAyF;QACzF,+DAA+D;QAC/D,MAAMuC,SAAS/B,eAAeT;QAC9B,IAAIwC,WAAW,QAAQA,WAAWtB,UAAU;YAC1Cc,QAAQC,KAAK,CAAC;YACdN,SAAS;YACT3B,GAAGkC,KAAK;YACRC,WAAWlC;YACX,OAAOqB,QAAQrB;QACjB;QAEA,MAAMkB,aAAaF,MAAMhB,KAAKiB;QAE9B,IAAIuB,kBAA4B,EAAE;QAClC,IAAIV,qBAAqB;YACvBU,kBAAkB,MAAMlD,oBAAoB0B,MAAMhB,KAAKA,IAAIuB,OAAO;YAClE,MAAMtC,QAAQ+B,MAAM,YAAYa;QAClC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMY,gBAAgBC,QAAQ,OAAA,MAAM1D,QAAQgC,MAAM,iCAApB,kBAAA,OAA4C;QAC1EjB,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAEsC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMxD,UAAUyB,MAAMhB,KAAKA,IAAIuB,OAAO;QAEnE,OAAO;YAAExB;YAAIiB;YAAMhB;YAAKyB;YAAQqB;YAAQC,UAAU;mBAAIP;mBAAoBO;aAAS;QAAC;IACtF,EAAE,OAAOzC,KAAK;QACZ,IAAI,CAACoB,QAAQ3B,GAAGkC,KAAK;QACrB,MAAM3B;IACR;AACF;AAEA,wFAAwF;AACxF,qDAAqD;AACrD,OAAO,eAAe0C,WAAWhD,GAAmB;IAClD,MAAM,EAAED,EAAE,EAAEiB,IAAI,EAAEhB,KAAKiD,WAAW,EAAExB,MAAM,EAAEqB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAM1B,QAAQrB;IAC/E,OAAO;QAAEkD,OAAOxD,YAAYK,IAAIiB,MAAMiC,aAAaA,YAAY1B,OAAO;QAAGvB,KAAKiD;QAAaxB;QAAQqB;QAAQC;IAAS;AACtH;AAEA,OAAO,eAAeI,SAASD,KAAY;IACzC,MAAME,OAAO,MAAMF,MAAMxC,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAM0C,KAAKzC,GAAG,EAAC,EAAqB0C,CAAC;AAChD;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAASnB,WAAWlC,GAAmB;IAC5CzB,OAAOC,KAAKwB,IAAIuB,OAAO,EAAE3C,YAAY;QAAE4C,WAAW;QAAM8B,OAAO;IAAK;AACtE"}

package/dist/esm/store/sqlite/store.js

@@ -1,9 +1,11 @@
 import { contentTokenize } from '../../config/index.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
+import { hasVectorRow, pendingRows } from '../vectors.js';
+import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
 import { reconcile } from './reconcile.js';
-import { hasVectorRow, pendingRows, scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
+import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 export const CAPABILITIES = new Set([
     'phrases',
     'snippets',
@@ -38,6 +40,9 @@ export function createStore(db, conn, cfg, baseDir) {
                 return [
                     ...await getColumns(conn)
                 ];
+            },
+            async fieldStats (columns, scopeWhere) {
+                return fieldStats(conn, columns, scopeWhere);
             }
         },
         lexical: {
@@ -62,6 +67,14 @@ export function createStore(db, conn, cfg, baseDir) {
                 return hasVectorRow(conn, path);
             }
         },
+        async engineStatus () {
+            // Read back rather than recomputed: this is what open() actually set (3x the largest
+            // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.
+            const row = db.prepare('PRAGMA busy_timeout').get();
+            return {
+                busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)`
+            };
+        },
         raw: {
             async prepare (sql) {
                 const stmt = db.prepare(sql);

package/dist/esm/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { hasVectorRow, pendingRows, scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth (see\n// transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","getColumns","withTransaction","queryLexical","reconcile","hasVectorRow","pendingRows","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,YAAY,EAAEC,WAAW,EAAEC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAExG,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU,EAAE;AAEjI,0FAA0F;AAC1F,wFAAwF;AACxF,iEAAiE;AACjE,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOvB,gBAAgBY,MAAMW;QAC/B;QACA,MAAMrB;YACJ,OAAOA,UAAUU,MAAMC,KAAKC;QAC9B;QACAU,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM1B,WAAWa;iBAAO;YACtC;QACF;QACAc,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAO5B,aAAaW,MAAMgB,OAAOC,MAAM/B,gBAAgBe,SAASiB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAO5B,YAAYQ;YACrB;YACA,MAAMqB,cAAaC,IAAsB;gBACvC,MAAM3B,iBAAiBK,MAAMsB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOlC,eAAeO,MAAMwB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOvB,YAAYM,MAAM6B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAOtC,aAAaS,MAAM6B;YAC5B;QACF;QACAE,KAAK;YACH,MAAMxB,SAAQD,GAAW;gBACvB,MAAM0B,OAAOjC,GAAGQ,OAAO,CAACD;gBACxB0B,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACLpB,SAAS,IAAMmB,KAAKnB,OAAO;oBAC3B,2EAA2E;oBAC3EqB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJrC,GAAGqC,KAAK;QACV;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth (see\n// transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU,EAAE;AAEjI,0FAA0F;AAC1F,wFAAwF;AACxF,iEAAiE;AACjE,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAMnB;YACJ,OAAOA,UAAUQ,MAAMC,KAAKC;QAC9B;QACAU,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACA,MAAMV,YAAWuB,OAAO,EAAEC,UAAU;gBAClC,OAAOxB,WAAWU,MAAMa,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAO3B,aAAaS,MAAMiB,OAAOC,MAAMjC,gBAAgBgB,SAASkB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOhC,YAAYW;YACrB;YACA,MAAMsB,cAAaC,IAAsB;gBACvC,MAAM5B,iBAAiBK,MAAMuB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOnC,eAAeO,MAAMyB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOxB,YAAYM,MAAM8B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAO1C,aAAaY,MAAM8B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAMlC,GAAGQ,OAAO,CAAC,uBAAuB2B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM9B,SAAQD,GAAW;gBACvB,MAAMgC,OAAOvC,GAAGQ,OAAO,CAACD;gBACxBgC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACL1B,SAAS,IAAMyB,KAAKzB,OAAO;oBAC3B,2EAA2E;oBAC3E2B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJ3C,GAAG2C,KAAK;QACV;IACF;AACF"}

package/dist/esm/store/sqlite/vectors.d.ts

@@ -1,8 +1,4 @@
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
-export declare function pendingRows(conn: Connection): Promise<Array<{
-    path: string;
-    chunk: number;
-}>>;
 export declare function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(conn: Connection, path: string, opts: {
@@ -10,4 +6,3 @@ export declare function scanSimilar(conn: Connection, path: string, opts: {
     allowed?: Set<string>;
     k: number;
 }): Promise<VectorSimilar[]>;
-export declare function hasVectorRow(conn: Connection, path: string): Promise<boolean>;