sensemaking 0.23.0 → 0.24.0

package/dist/cjs/store/duckdb/vectors.js

@@ -302,7 +302,7 @@ function writeVectorBatch(duckdb, conn, dims, rows) {
 }
 function scanCandidates(duckdb, qv, dims, fetch, allowed) {
     return _async_to_generator(function() {
-        var _ref, arrayValue, ARRAY, DOUBLE, allowedList, sql, stmt, values, types, reader, rows;
+        var _ref, arrayValue, ARRAY, DOUBLE, allowedList, query, queryNorm, score, sql, stmt, values, types, reader, rows;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -317,7 +317,12 @@ function scanCandidates(duckdb, qv, dims, fetch, allowed) {
                 case 1:
                     _ref = _state.sent(), arrayValue = _ref.arrayValue, ARRAY = _ref.ARRAY, DOUBLE = _ref.DOUBLE;
                     allowedList = allowed ? _to_consumable_array(allowed) : [];
-                    sql = 'SELECT "path", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT "path", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL '.concat(allowed ? "AND ".concat(inClause('"path"', allowedList.length)) : '', '\n    ) sub\n    GROUP BY "path"\n    ORDER BY score DESC\n    LIMIT ?');
+                    query = padded(qv, dims);
+                    queryNorm = Math.sqrt(query.reduce(function(sum, value) {
+                        return sum + value * value;
+                    }, 0));
+                    score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';
+                    sql = 'SELECT "path", start_line, end_line, score\n    FROM (\n      SELECT "path", start_line, end_line, score,\n        row_number() OVER (PARTITION BY "path" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT "path", chunk, start_line, end_line, '.concat(score, " AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ").concat(allowed ? "AND ".concat(inClause('"path"', allowedList.length)) : '', '\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, "path" ASC\n    LIMIT ?');
                     return [
                         4,
                         duckdb.prepare(sql)
@@ -332,14 +337,14 @@ function scanCandidates(duckdb, qv, dims, fetch, allowed) {
                         5,
                         6
                     ]);
-                    values = [
-                        arrayValue(padded(qv, dims))
-                    ].concat(_to_consumable_array(allowedList), [
+                    values = _to_consumable_array(queryNorm === 0 ? [] : [
+                        arrayValue(query)
+                    ]).concat(_to_consumable_array(allowedList), [
                         fetch
                     ]);
-                    types = [
+                    types = _to_consumable_array(queryNorm === 0 ? [] : [
                         ARRAY(DOUBLE, dims)
-                    ].concat(_to_consumable_array(allowedList.map(function() {
+                    ]).concat(_to_consumable_array(allowedList.map(function() {
                         return untyped;
                     })), [
                         untyped
@@ -414,7 +419,7 @@ function scanSimilar(duckdb, conn, dims, path, opts) {
                     allowedList = opts.allowed ? _to_consumable_array(opts.allowed) : [];
                     sql = "WITH targets(tv) AS (VALUES ".concat(targets.map(function() {
                         return '(?)';
-                    }).join(', '), ')\n    SELECT e."path" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e."path" != ?\n      ').concat(excludeList.length > 0 ? "AND NOT ".concat(inClause('e."path"', excludeList.length)) : '', "\n      ").concat(opts.allowed ? "AND ".concat(inClause('e."path"', allowedList.length)) : '', '\n    GROUP BY e."path"\n    ORDER BY score DESC\n    LIMIT ?');
+                    }).join(', '), '), grouped AS (\n    SELECT e."path" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e."path" != ?\n      ').concat(excludeList.length > 0 ? "AND NOT ".concat(inClause('e."path"', excludeList.length)) : '', "\n      ").concat(opts.allowed ? "AND ".concat(inClause('e."path"', allowedList.length)) : '', '\n    GROUP BY e."path"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?');
                     return [
                         4,
                         duckdb.prepare(sql)

package/dist/cjs/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks the winning chunk's line range from the same row max(score) came from, per path.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","duckdbApi","rewriteUpdate","Error","withTransaction","stmt","types","prepare","sql","push","path","chunk","bind","run","destroySync","qv","fetch","allowed","allowedList","reader","size","map","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","score","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkEsBA;eAAAA;;QA4BAC;eAAAA;;QArDAC;eAAAA;;;6BAxCU;yBAEO;uBACT;wBACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,IAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,IAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCtB;AAEJ;AAIO,SAAeL,iBAAiB4B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEyB,IAAsB;;YAE/E,MAA9BC,YAAYC,OAAOC,QACrBC;;;;oBAFN,IAAIJ,KAAKlB,MAAM,KAAK,GAAG;;;oBACe;;wBAAMuB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBC,YAAYE,IAAAA,sBAAa,EAAC,mEAAmEN,KAAKlB,MAAM;oBAC9G,IAAI,CAACsB,WAAW,MAAM,IAAIG,MAAM;oBAEhC;;wBAAMC,IAAAA,8BAAe,EAACT,MAAM;;oCACpBU,MAEEnC,QACAoC,OACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB;;;;4CAJA;;gDAAMS,OAAOa,OAAO,CAACP,UAAUQ,GAAG;;;4CAAzCH,OAAO;;;;;;;;;4CAELnC;4CACAoC;4CACD,kCAAA,2BAAA;;gDAAL,IAAK,YAAaV,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;oDAAbX,MAAN;oDACHf,OAAOuC,IAAI,CAACZ,WAAWb,WAAWC,KAAKd,QAAQc,IAAIyB,IAAI,EAAEzB,IAAI0B,KAAK;oDAClEL,MAAMG,IAAI,CAACX,MAAMC,QAAQ5B,OAAOJ,SAASA;gDAC3C;;gDAHK;gDAAA;;;yDAAA,6BAAA;wDAAA;;;wDAAA;8DAAA;;;;4CAILsC,KAAKO,IAAI,CAAC1C,QAAQoC;4CAClB;;gDAAMD,KAAKQ,GAAG;;;4CAAd;;;;;;4CAEAR,KAAKS,WAAW;;;;;;;;;;4BAEpB;;;;oBAdA;;;;;;IAeF;;AAIO,SAAelD,eAAe8B,MAAwB,EAAEqB,EAAgB,EAAE5C,IAAY,EAAE6C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BpB,YAAYC,OAAOC,QACrBmB,aACAV,KASAH,MAEEnC,QACAoC,OAEAa,QACAvB;;;;oBAlBR,IAAIqB,WAAWA,QAAQG,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMnB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBmB,cAAcD,UAAW,qBAAGA;oBAC5BT,MAAM,AAAC,iRAIiF,OAA/DS,UAAU,AAAC,OAA6C,OAAvCtC,SAAS,UAAUuC,YAAYxC,MAAM,KAAM,IAAG;oBAKjF;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB;wBAAC2B,WAAW5B,OAAO8C,IAAI5C;sBAAvB,OAA+B,qBAAG+C;wBAAaF;;oBACvEV,QAAsB;wBAACR,MAAMC,QAAQ5B;sBAAf,OAAsB,qBAAG+C,YAAYG,GAAG,CAAC;+BAAMtD;;wBAAUA;;oBACrFsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKiB,aAAa;;;oBAAjCH,SAAS;oBACTvB,OAAOuB,OAAOI,eAAe;oBACnC;;wBAAO3B,KAAKyB,GAAG,CAAC,SAACG;mCAAO;gCAAEd,MAAMc,EAAEd,IAAI;gCAAEe,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEM,KAAK;4BAAE;;;;oBAE/GzB,KAAKS,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAejD,YAAY6B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEuC,IAAY,EAAEqB,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9BrC,YAAYC,OAAOC,QACrBoC,aACAjB,aACAV,KASAH,MAEEnC,QACAoC,OAEAa,QACAvB;;;;oBAxBR,IAAImC,KAAKd,OAAO,IAAIc,KAAKd,OAAO,CAACG,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAMzB,KAAKY,OAAO,CAAC;;;oBAAhCyB,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAC1B;;;oBAAnCuB,aAAc;oBACpB,IAAIA,WAAWvD,MAAM,KAAK,GAAG;;;;oBACvBwD,UAAUG,IAAAA,uBAAY,EAACJ,YAAYZ,GAAG,CAAC,SAACpC;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMa,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBoC,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9BpB,cAAca,KAAKd,OAAO,GAAI,qBAAGc,KAAKd,OAAO;oBAC7CT,MAAM,AAAC,+BAIP2B,OAJqCD,QAAQb,GAAG,CAAC;+BAAM;uBAAOtC,IAAI,CAAC,OAAM,qLAKzEgD,OADAI,YAAYzD,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAYwD,YAAYzD,MAAM,KAAM,IAAG,YACd,OAAtEqD,KAAKd,OAAO,GAAG,AAAC,OAA+C,OAAzCtC,SAAS,YAAYuC,YAAYxC,MAAM,KAAM,IAAG;oBAI/D;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB,AAAC,qBAAGgE,QAAQb,GAAG,CAAC,SAACkB;+BAAM1C,WAAW5B,OAAOsE,GAAGpE;;wBAASuC;uBAAM,qBAAGyB,cAAa,qBAAGjB;wBAAaa,KAAKS,CAAC;;oBACzHlC,QAAsB,AAAC,qBAAG4B,QAAQb,GAAG,CAAC;+BAAMvB,MAAMC,QAAQ5B;;wBAAQJ;uBAAS,qBAAGoE,YAAYd,GAAG,CAAC;+BAAMtD;yBAAU,qBAAGmD,YAAYG,GAAG,CAAC;+BAAMtD;;wBAAUA;;oBACvJsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKiB,aAAa;;;oBAAjCH,SAAS;oBACTvB,OAAOuB,OAAOI,eAAe;oBACnC;;wBAAO3B,KAAKyB,GAAG,CAAC,SAACG;mCAAO;gCAAEd,MAAMc,EAAEd,IAAI;gCAAEkB,YAAYC,IAAAA,mBAAQ,EAACL,EAAEM,KAAK;4BAAE;;;;oBAEtEzB,KAAKS,WAAW;;;;;;;;;;IAEpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","duckdbApi","rewriteUpdate","Error","withTransaction","stmt","types","prepare","sql","push","path","chunk","bind","run","destroySync","qv","fetch","allowed","allowedList","query","queryNorm","score","reader","size","sqrt","reduce","sum","value","map","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkEsBA;eAAAA;;QAmCAC;eAAAA;;QA5DAC;eAAAA;;;6BAxCU;yBAEO;uBACT;wBACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,IAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,IAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCtB;AAEJ;AAIO,SAAeL,iBAAiB4B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEyB,IAAsB;;YAE/E,MAA9BC,YAAYC,OAAOC,QACrBC;;;;oBAFN,IAAIJ,KAAKlB,MAAM,KAAK,GAAG;;;oBACe;;wBAAMuB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBC,YAAYE,IAAAA,sBAAa,EAAC,mEAAmEN,KAAKlB,MAAM;oBAC9G,IAAI,CAACsB,WAAW,MAAM,IAAIG,MAAM;oBAEhC;;wBAAMC,IAAAA,8BAAe,EAACT,MAAM;;oCACpBU,MAEEnC,QACAoC,OACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB;;;;4CAJA;;gDAAMS,OAAOa,OAAO,CAACP,UAAUQ,GAAG;;;4CAAzCH,OAAO;;;;;;;;;4CAELnC;4CACAoC;4CACD,kCAAA,2BAAA;;gDAAL,IAAK,YAAaV,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;oDAAbX,MAAN;oDACHf,OAAOuC,IAAI,CAACZ,WAAWb,WAAWC,KAAKd,QAAQc,IAAIyB,IAAI,EAAEzB,IAAI0B,KAAK;oDAClEL,MAAMG,IAAI,CAACX,MAAMC,QAAQ5B,OAAOJ,SAASA;gDAC3C;;gDAHK;gDAAA;;;yDAAA,6BAAA;wDAAA;;;wDAAA;8DAAA;;;;4CAILsC,KAAKO,IAAI,CAAC1C,QAAQoC;4CAClB;;gDAAMD,KAAKQ,GAAG;;;4CAAd;;;;;;4CAEAR,KAAKS,WAAW;;;;;;;;;;4BAEpB;;;;oBAdA;;;;;;IAeF;;AAIO,SAAelD,eAAe8B,MAAwB,EAAEqB,EAAgB,EAAE5C,IAAY,EAAE6C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BpB,YAAYC,OAAOC,QACrBmB,aACAC,OACAC,WACAC,OACAb,KAaAH,MAEEnC,QACAoC,OAEAgB,QACA1B;;;;oBAzBR,IAAIqB,WAAWA,QAAQM,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMtB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBmB,cAAcD,UAAW,qBAAGA;oBAC5BE,QAAQlD,OAAO8C,IAAI5C;oBACnBiD,YAAY5C,KAAKgD,IAAI,CAACL,MAAMM,MAAM,CAAC,SAACC,KAAKC;+BAAUD,MAAMC,QAAQA;uBAAO;oBACxEN,QAAQD,cAAc,IAAI,QAAQ;oBAClCZ,MAAM,AAAC,4QAOoBS,OAFmBI,OAAM,yEAEsC,OAA/DJ,UAAU,AAAC,OAA6C,OAAvCtC,SAAS,UAAUuC,YAAYxC,MAAM,KAAM,IAAG;oBAMnF;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB,AAAC,qBAAIkD,cAAc;wBAAUvB,WAAWsB;8BAAU,qBAAGD;wBAAaF;;oBAC1FV,QAAsB,AAAC,qBAAIc,cAAc;wBAAUtB,MAAMC,QAAQ5B;8BAAS,qBAAG+C,YAAYU,GAAG,CAAC;+BAAM7D;;wBAAUA;;oBACnHsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKwB,aAAa;;;oBAAjCP,SAAS;oBACT1B,OAAO0B,OAAOQ,eAAe;oBACnC;;wBAAOlC,KAAKgC,GAAG,CAAC,SAACG;mCAAO;gCAAErB,MAAMqB,EAAErB,IAAI;gCAAEsB,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEV,KAAK;4BAAE;;;;oBAE/GhB,KAAKS,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAejD,YAAY6B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEuC,IAAY,EAAE2B,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9B3C,YAAYC,OAAOC,QACrB0C,aACAvB,aACAV,KAWAH,MAEEnC,QACAoC,OAEAgB,QACA1B;;;;oBA1BR,IAAIyC,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACM,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAM5B,KAAKY,OAAO,CAAC;;;oBAAhC+B,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAChC;;;oBAAnC6B,aAAc;oBACpB,IAAIA,WAAW7D,MAAM,KAAK,GAAG;;;;oBACvB8D,UAAUG,IAAAA,uBAAY,EAACJ,YAAYX,GAAG,CAAC,SAAC3C;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMa,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrB0C,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9B1B,cAAcmB,KAAKpB,OAAO,GAAI,qBAAGoB,KAAKpB,OAAO;oBAC7CT,MAAM,AAAC,+BAIPiC,OAJqCD,QAAQZ,GAAG,CAAC;+BAAM;uBAAO7C,IAAI,CAAC,OAAM,kTAKzEsD,OADAI,YAAY/D,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAY8D,YAAY/D,MAAM,KAAM,IAAG,YACd,OAAtE2D,KAAKpB,OAAO,GAAG,AAAC,OAA+C,OAAzCtC,SAAS,YAAYuC,YAAYxC,MAAM,KAAM,IAAG;oBAM/D;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB,AAAC,qBAAGsE,QAAQZ,GAAG,CAAC,SAACiB;+BAAMhD,WAAW5B,OAAO4E,GAAG1E;;wBAASuC;uBAAM,qBAAG+B,cAAa,qBAAGvB;wBAAamB,KAAKS,CAAC;;oBACzHxC,QAAsB,AAAC,qBAAGkC,QAAQZ,GAAG,CAAC;+BAAM9B,MAAMC,QAAQ5B;;wBAAQJ;uBAAS,qBAAG0E,YAAYb,GAAG,CAAC;+BAAM7D;yBAAU,qBAAGmD,YAAYU,GAAG,CAAC;+BAAM7D;;wBAAUA;;oBACvJsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKwB,aAAa;;;oBAAjCP,SAAS;oBACT1B,OAAO0B,OAAOQ,eAAe;oBACnC;;wBAAOlC,KAAKgC,GAAG,CAAC,SAACG;mCAAO;gCAAErB,MAAMqB,EAAErB,IAAI;gCAAEyB,YAAYC,IAAAA,mBAAQ,EAACL,EAAEV,KAAK;4BAAE;;;;oBAEtEhB,KAAKS,WAAW;;;;;;;;;;IAEpB"}

package/dist/cjs/store/native.js

@@ -242,15 +242,16 @@ function installOnce(specifier, nodeModulesPath) {
     return promise;
 }
 // Node caches a bare specifier's resolution, including a "not found" miss, for the process
-// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.
-function resolveEntryUrl(importName, nodeModulesPath) {
+// lifetime; use the installed entry directly, as a URL for ESM and a path for CJS.
+function resolveEntrySpecifier(importName, nodeModulesPath) {
     var _pkg_main;
     var pkgDir = _nodepath.join.apply(void 0, [
         nodeModulesPath
     ].concat(_to_consumable_array(importName.split('/'))));
     var pkg = JSON.parse((0, _nodefs.readFileSync)((0, _nodepath.join)(pkgDir, 'package.json'), 'utf8'));
     var entry = typeof pkg.exports === 'string' ? pkg.exports : (_pkg_main = pkg.main) !== null && _pkg_main !== void 0 ? _pkg_main : 'index.js';
-    return (0, _nodeurl.pathToFileURL)((0, _nodepath.join)(pkgDir, entry)).href;
+    var entryPath = (0, _nodepath.join)(pkgDir, entry);
+    return typeof require === 'undefined' ? (0, _nodeurl.pathToFileURL)(entryPath).href : entryPath;
 }
 function loadOrInstall(_0, _1) {
     return _async_to_generator(function(descriptor, nodeModulesPath) {
@@ -312,7 +313,7 @@ function loadOrInstall(_0, _1) {
                     ]);
                     return [
                         4,
-                        Promise.resolve(resolveEntryUrl(importName, nodeModulesPath)).then(function(p) {
+                        Promise.resolve(resolveEntrySpecifier(importName, nodeModulesPath)).then(function(p) {
                             return /*#__PURE__*/ _interop_require_wildcard(require(p));
                         })
                     ];

package/dist/cjs/store/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["loadOrInstall","packageNodeModules","_require","require","Module","createRequire","_dirname","dirname","__filename","fileURLToPath","dir","candidate","join","existsSync","pkg","JSON","parse","readFileSync","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","pathToFileURL","href","descriptor","installErr","loadErr","console","error","store","sizeHint","SenseError","message","process","platform","arch"],"mappings":";;;;;;;;;;;QAkEsBA;eAAAA;;QA9CNC;eAAAA;;;sBApByB;iEACtB;wBACW;uBACe;wBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAS3B,+FAA+F;AAC/F,sGAAsG;AACtG,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAC1F,IAAMG,WAAWC,IAAAA,iBAAO,EAAC,OAAOC,eAAe,cAAcA,aAAaC,IAAAA,sBAAa,EAAC;AAIjF,SAASR;IACd,IAAIS,MAAMJ;IACV,OAAS;QACP,IAAMK,YAAYC,IAAAA,cAAI,EAACF,KAAK;QAC5B,IAAIG,IAAAA,kBAAU,EAACF,YAAY;YACzB,IAAMG,MAAMC,KAAKC,KAAK,CAACC,IAAAA,oBAAY,EAACN,WAAW;YAC/C,IAAIG,IAAII,IAAI,KAAK,eAAe,OAAON,IAAAA,cAAI,EAACF,KAAK;QACnD;QACA,IAAMS,SAASZ,IAAAA,iBAAO,EAACG;QACvB,IAAIS,WAAWT,KAAK,MAAM,IAAIU,MAAM;QACpCV,MAAMS;IACR;AACF;AAEA,IAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,IAAMC,MAAM,AAAC,GAAgBD,OAAdD,WAAU,MAAoB,OAAhBC;IAC7B,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,IAAME,sBAAsB,AAAC3B,SAAS,yBAA0G4B,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,YAAO,GACP,SAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,kGAAkG;AAClG,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHX;IAF/D,IAAMuB,SAASzB,cAAI,OAAJA,KAAAA,GAAAA;QAAKa;KAA0C,CAA/Cb,OAAsB,qBAAGwB,WAAWE,KAAK,CAAC;IACzD,IAAMxB,MAAMC,KAAKC,KAAK,CAACC,IAAAA,oBAAY,EAACL,IAAAA,cAAI,EAACyB,QAAQ,iBAAiB;IAClE,IAAME,QAAQ,OAAOzB,IAAI0B,OAAO,KAAK,WAAW1B,IAAI0B,OAAO,IAAI1B,YAAAA,IAAI2B,IAAI,cAAR3B,uBAAAA,YAAY;IAC3E,OAAO4B,IAAAA,sBAAa,EAAC9B,IAAAA,cAAI,EAACyB,QAAQE,QAAQI,IAAI;AAChD;AAIO,SAAe3C;wCAAiB4C,UAA4B,EAAEnB,eAAuB;YAAEW,oBAOjFS,YAQAC;;;;;oBAfiFV,aAAAA,oEAAqBQ,WAAW9B,GAAG;;;;;;;;;oBAErH;;wBAAM,gBAAOsB;2EAAP;;;;oBAAd;;wBAAQ;;;;oBAERW,QAAQC,KAAK,CAAC,AAAC,iBAA2CJ,OAA3BA,WAAWK,KAAK,EAAC,YAAmEL,OAAzDA,WAAW9B,GAAG,EAAC,4CAA8D,OAApB8B,WAAWM,QAAQ,EAAC;;;;;;;;;oBAErI;;wBAAM3B,YAAYa,YAAYX;;;oBAA9B;;;;;;oBACOoB;oBACP,MAAM,IAAIM,oBAAU,CAClB,4BACA,AAAC,UAAoCP,OAA3BA,WAAWK,KAAK,EAAC,YAAqE,OAA3DL,WAAW9B,GAAG,EAAC,8CAAkP8B,OAAtM,AAACC,WAAqBO,OAAO,EAAC,yKAAuL,OAAfR,WAAW9B,GAAG,EAAC;;;;;;;;oBAI/S;;wBAAM,gBAAOqB,gBAAgBC,YAAYX;2EAAnC;;;;oBAAd;;wBAAQ;;;oBACDqB;oBACP,MAAM,IAAIK,oBAAU,CAClB,4BACA,AAAC,UAAwCP,OAA/BA,WAAWK,KAAK,EAAC,gBAA4D,OAA9CL,WAAW9B,GAAG,EAAC,iCAA0GuC,OAA3E,AAACP,QAAkBM,OAAO,EAAC,kDAAoEC,OAApBA,QAAQC,QAAQ,EAAC,KAAyCV,OAAtCS,QAAQE,IAAI,EAAC,0BAAwC,OAAfX,WAAW9B,GAAG,EAAC;;;;;;;;;;;;IAInP"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; use the installed entry directly, as a URL for ESM and a path for CJS.\nfunction resolveEntrySpecifier(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  const entryPath = join(pkgDir, entry);\n  return typeof require === 'undefined' ? pathToFileURL(entryPath).href : entryPath;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntrySpecifier(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["loadOrInstall","packageNodeModules","_require","require","Module","createRequire","_dirname","dirname","__filename","fileURLToPath","dir","candidate","join","existsSync","pkg","JSON","parse","readFileSync","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntrySpecifier","importName","pkgDir","split","entry","exports","main","entryPath","pathToFileURL","href","descriptor","installErr","loadErr","console","error","store","sizeHint","SenseError","message","process","platform","arch"],"mappings":";;;;;;;;;;;QAmEsBA;eAAAA;;QA/CNC;eAAAA;;;sBApByB;iEACtB;wBACW;uBACe;wBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAS3B,+FAA+F;AAC/F,sGAAsG;AACtG,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAC1F,IAAMG,WAAWC,IAAAA,iBAAO,EAAC,OAAOC,eAAe,cAAcA,aAAaC,IAAAA,sBAAa,EAAC;AAIjF,SAASR;IACd,IAAIS,MAAMJ;IACV,OAAS;QACP,IAAMK,YAAYC,IAAAA,cAAI,EAACF,KAAK;QAC5B,IAAIG,IAAAA,kBAAU,EAACF,YAAY;YACzB,IAAMG,MAAMC,KAAKC,KAAK,CAACC,IAAAA,oBAAY,EAACN,WAAW;YAC/C,IAAIG,IAAII,IAAI,KAAK,eAAe,OAAON,IAAAA,cAAI,EAACF,KAAK;QACnD;QACA,IAAMS,SAASZ,IAAAA,iBAAO,EAACG;QACvB,IAAIS,WAAWT,KAAK,MAAM,IAAIU,MAAM;QACpCV,MAAMS;IACR;AACF;AAEA,IAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,IAAMC,MAAM,AAAC,GAAgBD,OAAdD,WAAU,MAAoB,OAAhBC;IAC7B,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,IAAME,sBAAsB,AAAC3B,SAAS,yBAA0G4B,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,YAAO,GACP,SAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,mFAAmF;AACnF,SAASQ,sBAAsBC,UAAkB,EAAEX,eAAuB;QAGTX;IAF/D,IAAMuB,SAASzB,cAAI,OAAJA,KAAAA,GAAAA;QAAKa;KAA0C,CAA/Cb,OAAsB,qBAAGwB,WAAWE,KAAK,CAAC;IACzD,IAAMxB,MAAMC,KAAKC,KAAK,CAACC,IAAAA,oBAAY,EAACL,IAAAA,cAAI,EAACyB,QAAQ,iBAAiB;IAClE,IAAME,QAAQ,OAAOzB,IAAI0B,OAAO,KAAK,WAAW1B,IAAI0B,OAAO,IAAI1B,YAAAA,IAAI2B,IAAI,cAAR3B,uBAAAA,YAAY;IAC3E,IAAM4B,YAAY9B,IAAAA,cAAI,EAACyB,QAAQE;IAC/B,OAAO,OAAOpC,YAAY,cAAcwC,IAAAA,sBAAa,EAACD,WAAWE,IAAI,GAAGF;AAC1E;AAIO,SAAe1C;wCAAiB6C,UAA4B,EAAEpB,eAAuB;YAAEW,oBAOjFU,YAQAC;;;;;oBAfiFX,aAAAA,oEAAqBS,WAAW/B,GAAG;;;;;;;;;oBAErH;;wBAAM,gBAAOsB;2EAAP;;;;oBAAd;;wBAAQ;;;;oBAERY,QAAQC,KAAK,CAAC,AAAC,iBAA2CJ,OAA3BA,WAAWK,KAAK,EAAC,YAAmEL,OAAzDA,WAAW/B,GAAG,EAAC,4CAA8D,OAApB+B,WAAWM,QAAQ,EAAC;;;;;;;;;oBAErI;;wBAAM5B,YAAYa,YAAYX;;;oBAA9B;;;;;;oBACOqB;oBACP,MAAM,IAAIM,oBAAU,CAClB,4BACA,AAAC,UAAoCP,OAA3BA,WAAWK,KAAK,EAAC,YAAqE,OAA3DL,WAAW/B,GAAG,EAAC,8CAAkP+B,OAAtM,AAACC,WAAqBO,OAAO,EAAC,yKAAuL,OAAfR,WAAW/B,GAAG,EAAC;;;;;;;;oBAI/S;;wBAAM,gBAAOqB,sBAAsBC,YAAYX;2EAAzC;;;;oBAAd;;wBAAQ;;;oBACDsB;oBACP,MAAM,IAAIK,oBAAU,CAClB,4BACA,AAAC,UAAwCP,OAA/BA,WAAWK,KAAK,EAAC,gBAA4D,OAA9CL,WAAW/B,GAAG,EAAC,iCAA0GwC,OAA3E,AAACP,QAAkBM,OAAO,EAAC,kDAAoEC,OAApBA,QAAQC,QAAQ,EAAC,KAAyCV,OAAtCS,QAAQE,IAAI,EAAC,0BAAwC,OAAfX,WAAW/B,GAAG,EAAC;;;;;;;;;;;;IAInP"}

package/dist/cjs/store/open.js

@@ -16,6 +16,7 @@ _export(exports, {
         return openWithDialect;
     }
 });
+var _nodediagnostics_channel = require("node:diagnostics_channel");
 var _nodefs = require("node:fs");
 var _nodepath = require("node:path");
 var _indexts = require("../config/index.js");
@@ -158,6 +159,7 @@ function _ts_generator(thisArg, body) {
 // duckdb/turso hold the cache file for their connection's whole life, so a second command waits.
 // Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.
 var LOCK_POLL_MS = 50;
+var lockWait = (0, _nodediagnostics_channel.channel)('sensemaking.store.lock-wait');
 function connectUnlocked(dbPath, cfg, dialect) {
     return _async_to_generator(function() {
         var budgetMs, deadline, err, _dialect_isLocked;
@@ -186,6 +188,11 @@ function connectUnlocked(dbPath, cfg, dialect) {
                 case 3:
                     err = _state.sent();
                     if (!((_dialect_isLocked = dialect.isLocked) === null || _dialect_isLocked === void 0 ? void 0 : _dialect_isLocked.call(dialect, err))) throw err;
+                    // Observe a real rejected native connection, never merely the start of an async open.
+                    if (lockWait.hasSubscribers) lockWait.publish({
+                        store: cfg.store,
+                        dbPath: dbPath
+                    });
                     if (Date.now() >= deadline) {
                         throw new _errorsts.SenseError('STORE_BUSY', "another sense process is using this tree's ".concat(cfg.store, " cache (").concat(dbPath, ") and did not release it within ").concat(Math.round(budgetMs / 1000), 's; wait for that command to finish, or set "store" to "sqlite" in sense.config.json, which serves concurrent commands'));
                     }

package/dist/cjs/store/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(cfg.baseDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["docCount","openWithDialect","LOCK_POLL_MS","connectUnlocked","dbPath","cfg","dialect","budgetMs","deadline","err","lockWaitBudgetMs","baseDir","Date","now","connect","isLocked","SenseError","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","stateDir","handle","conn","closed","builder","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","changedKeys","presetForced","embedKind","toggles","changed","recordedMaxMs","embedParsed","featureParsed","parsed","warnings","stages","join","STATE_DIR","mkdirSync","recursive","filename","createBuilder","reconcileDialect","exec","getMeta","featureSignature","FEATURES","schemaVersion","console","error","close","clearCache","changedSignatureKeys","isPresetOnlyChange","forcedPresetPaths","size","has","classifyEmbedChange","isFeatureOnlyChange","classifyFeatureToggles","embedIdentityAdopted","setMeta","signatureDiff","ensureSchema","setDerivedBusyTimeout","Number","min","max","invalidate","invalidateFeatures","build","resolvedCfg","createStore","rekeyChunkText","stmt","prepare","get","n"],"mappings":";;;;;;;;;;;QAgMsBA;eAAAA;;QARAC;eAAAA;;;sBAxLI;wBACL;uBAEuB;yBACb;wBACJ;wBACF;yBACK;uBACH;4BAES;8BAEwB;0BAC3B;6BACqB;wBACrB;2BACyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8B1E,iGAAiG;AACjG,oFAAoF;AACpF,IAAMC,eAAe;AAErB,SAAeC,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;;YAChGC,UACAC,UAIKC,KACFH;;;;oBANHC,WAAWG,IAAAA,4BAAgB,EAACL,IAAIM,OAAO;oBACvCH,WAAWI,KAAKC,GAAG,KAAKN;;;;;;;;;oBAGnB;;wBAAMD,QAAQQ,OAAO,CAACV,QAAQC;;;oBAArC;;wBAAO;;;oBACAI;oBACP,IAAI,GAACH,oBAAAA,QAAQS,QAAQ,cAAhBT,wCAAAA,uBAAAA,SAAmBG,OAAe,MAAMA;oBAC7C,IAAIG,KAAKC,GAAG,MAAML,UAAU;wBAC1B,MAAM,IAAIQ,oBAAU,CAAC,cAAc,AAAC,8CAAiEZ,OAApBC,IAAIY,KAAK,EAAC,YAAmDC,OAAzCd,QAAO,oCAA8D,OAA5Bc,KAAKC,KAAK,CAACZ,WAAW,OAAM;oBAC5K;oBACA;;wBAAM,IAAIa,QAAQ,SAACC;mCAAYC,WAAWD,SAASnB;;;;oBAAnD;;;;;;;;;;;;;;;;IAGN;;AAEA,SAAeqB,mBAA2BlB,GAAmB,EAAEC,OAA4B;;YACnFkB,UAEApB,QAEmB,MAAjBqB,QAAQC,MAGZC,QAGEC,SAMEC,SACAC,UACAC,cAGFC,aAEAC,iBAEAC,gBAYIC,aAIAC,cAGAC,WAIAC,SAkBEC,SASAA,UAIAA,UAcsB,OAAxBC,eAOFC,mBACAC,sBAE+B,OAA7BC,QAAQC,UAAUC,QAKnBpC;;;;oBA5GHe,WAAWsB,IAAAA,cAAI,EAACzC,IAAIM,OAAO,EAAEoC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAACxB,UAAU;wBAAEyB,WAAW;oBAAK;oBAChC7C,SAAS0C,IAAAA,cAAI,EAACtB,UAAUlB,QAAQ4C,QAAQ;oBAErB;;wBAAM/C,gBAAgBC,QAAQC,KAAKC;;;oBAAnC,OAAA,eAAjBmB,SAAiB,KAAjBA,QAAQC,OAAS,KAATA;oBAChB,gGAAgG;oBAChG,kGAAkG;oBAC9FC,SAAS;oBACb,+FAA+F;oBAC/F,2CAA2C;oBACrCC,UAAUuB,IAAAA,wBAAa,EAACzB,MAAMrB,KAAKA,IAAIM,OAAO,EAAEL,QAAQ8C,gBAAgB;;;;;;;;;oBAE5E;;wBAAM1B,KAAK2B,IAAI,CAAC;;;oBAAhB;oBAIgB;;wBAAMC,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA9BG,UAAU;oBACC;;wBAAMyB,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA/BI,WAAW;oBACXC,eAAewB,IAAAA,yBAAgB,EAAClD,KAAKmD,kBAAQ;yBAQ/C,CAAA,AAAC3B,YAAY,QAAQA,YAAYvB,QAAQmD,aAAa,IAAM3B,aAAa,QAAQA,aAAaC,YAAY,GAA1G;;;;yBAGEF,CAAAA,YAAY,QAAQA,YAAYvB,QAAQmD,aAAa,AAAD,GAApD5B;;;;oBACF6B,QAAQC,KAAK,CAAC;oBACd,kFAAkF;oBAClFhC,SAAS;oBACT;;wBAAMrB,QAAQsD,KAAK,CAACnC;;;oBAApB;oBACAoC,IAAAA,mBAAU,EAACxD;oBACX;;wBAAOkB,mBAAmBlB,KAAKC;;;oBAE3B6B,cAAc2B,IAAAA,iCAAoB,EAAChC,qBAAAA,sBAAAA,WAAY,IAAIC;oBACzD,oFAAoF;oBACpF,0FAA0F;oBAC1F,2EAA2E;oBACrEK,eAAe2B,IAAAA,iCAAkB,EAAC5B,eAAe6B,IAAAA,gCAAiB,EAAC3D,KAAKA,IAAIM,OAAO,EAAEmB,qBAAAA,sBAAAA,WAAY,IAAIK,eAAe;oBAC1H,yFAAyF;oBACzF,oFAAoF;oBAC9EE,YAAYF,YAAY8B,IAAI,KAAK,KAAK9B,YAAY+B,GAAG,CAAC,WAAWC,IAAAA,iCAAmB,EAACrC,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;oBAC3H,4FAA4F;oBAC5F,6FAA6F;oBAC7F,4DAA4D;oBACtDO,UAAU8B,IAAAA,mCAAmB,EAACjC,eAAekC,IAAAA,sCAAsB,EAACvC,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcI,eAAe;yBACnHA,CAAAA,YAAY8B,IAAI,KAAK,KAAK9B,YAAY+B,GAAG,CAAC,YAAYI,IAAAA,iCAAoB,EAACxC,qBAAAA,sBAAAA,WAAY,IAAIC,aAAY,GAAvGI;;;;oBACF,iFAAiF;oBACjF,sDAAsD;oBACtDuB,QAAQC,KAAK,CAAC;oBACd;;wBAAMY,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSM,CAAAA,cAAc,IAAG,GAAjBA;;;;oBACT,yFAAyF;oBACzF,iEAAiE;oBACjEJ,kBAAkBI;oBAClBqB,QAAQC,KAAK,CAACtB,cAAc,UAAU,6EAA6E;oBACnH;;wBAAMkC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSK,CAAAA,iBAAiB,IAAG,GAApBA;;;;oBACT,kFAAkF;oBAClF,uEAAuE;oBACvE,sFAAsF;oBACtF,qFAAqF;oBACrFJ,cAAcI;oBACRG,UAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,SAAQ;oBAC/C;;wBAAMgC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSO,CAAAA,YAAY,IAAG,GAAfA;;;;oBACT,uFAAuF;oBACvF,wFAAwF;oBACxF,kFAAkF;oBAClF,sDAAsD;oBACtDJ,iBAAiBI;oBACXC,WAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,UAAQ;oBAC/C;;wBAAMgC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;oBAEMQ,WAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,UAAQ;oBAC/CZ,SAAS;oBACT;;wBAAMrB,QAAQsD,KAAK,CAACnC;;;oBAApB;oBACAoC,IAAAA,mBAAU,EAACxD;oBACX;;wBAAOkB,mBAAmBlB,KAAKC;;;oBAInC;;wBAAMA,QAAQmE,YAAY,CAAChD,QAAQC,MAAMrB;;;oBAAzC;yBAIIC,QAAQoE,qBAAqB,EAA7BpE;;;;oBAC4B;;wBAAMgD,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA5Cc,gBAAgBmC;yBAAQ,QAAA,2BAAA,mBAAA,QAA4C;;oBAC1E;;wBAAMrE,QAAQoE,qBAAqB,CAACjD,QAAQC,MAAMR,KAAK0D,GAAG,CAAC1D,KAAK2D,GAAG,CAAC,OAAO,IAAIrC,gBAAgB;;;oBAA/F;;;yBAMkBP,iBAAAA;;;;oBAAmB;;wBAAML,QAAQkD,UAAU,CAAC7C;;;2BAA1B,AAAC,cAA2CU,MAAM;;;;;;2BAAG;;;oBAArFF;yBACgBP,gBAAAA;;;;oBAAkB;;wBAAMN,QAAQmD,kBAAkB,CAAC7C;;;4BAAlC,AAAC,cAAkDS,MAAM;;;;;;4BAAG;;;oBAA7FD;oBAE+B;;wBAAMd,QAAQoD,KAAK,CAAChD;;;oBAApB,QAAA,eAA7BW,SAA6B,MAA7BA,QAAQC,WAAqB,MAArBA,UAAUC,SAAW,MAAXA;oBAC1B,uIAAuI;oBACvI;;wBAAMjB,QAAQgC,KAAK;;;oBAAnB;oBAEA;;wBAAO;4BAAEnC,QAAAA;4BAAQC,MAAAA;4BAAMrB,KAAAA;4BAAKD,QAAAA;4BAAQuC,QAAQA,SAASF,cAAcC;4BAAeE,UAAAA;4BAAUC,QAAAA;wBAAO;;;oBAC5FpC;oBACP;;wBAAMmB,QAAQgC,KAAK;;;oBAAnB;yBACI,CAACjC,QAAD;;;;oBAAS;;wBAAMrB,QAAQsD,KAAK,CAACnC;;;oBAApB;;;oBACb,MAAMhB;;;;;;;IAEV;;AAIO,SAAeR,gBAAwBI,GAAmB,EAAEC,OAA4B;;YAChB,MAArEmB,QAAQC,MAAWuD,aAAa7E,QAAQuC,QAAQC,UAAUC,QAC5D5B;;;;oBADuE;;wBAAMM,mBAAmBlB,KAAKC;;;oBAA9B,OAAA,eAArEmB,SAAqE,KAArEA,QAAQC,OAA6D,KAA7DA,MAAWuD,cAAkD,KAAvD5E,KAAkBD,SAAqC,KAArCA,QAAQuC,SAA6B,KAA7BA,QAAQC,WAAqB,KAArBA,UAAUC,SAAW,KAAXA;oBAC5D5B,QAAQX,QAAQ4E,WAAW,CAACzD,QAAQC,MAAMuD;oBAChD,0FAA0F;oBAC1FE,IAAAA,yBAAc,EAACzD,MAAMT;oBACrB;;wBAAO;4BAAEA,OAAAA;4BAAOZ,KAAK4E;4BAAa7E,QAAAA;4BAAQuC,QAAAA;4BAAQC,UAAAA;4BAAUC,QAAAA;wBAAO;;;;IACrE;;AAEO,SAAe7C,SAASiB,KAAY;;YACnCmE;;;;oBAAO;;wBAAMnE,MAAMoE,OAAO,CAAC;;;oBAA3BD,OAAO;oBACJ;;wBAAMA,KAAKE,GAAG;;;oBAAvB;;wBAAS,cAAoCC,CAAC;;;;IAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { channel } from 'node:diagnostics_channel';\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\nconst lockWait = channel('sensemaking.store.lock-wait');\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(cfg.baseDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      // Observe a real rejected native connection, never merely the start of an async open.\n      if (lockWait.hasSubscribers) lockWait.publish({ store: cfg.store, dbPath });\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["docCount","openWithDialect","LOCK_POLL_MS","lockWait","channel","connectUnlocked","dbPath","cfg","dialect","budgetMs","deadline","err","lockWaitBudgetMs","baseDir","Date","now","connect","isLocked","hasSubscribers","publish","store","SenseError","Math","round","Promise","resolve","setTimeout","connectWithDialect","stateDir","handle","conn","closed","builder","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","changedKeys","presetForced","embedKind","toggles","changed","recordedMaxMs","embedParsed","featureParsed","parsed","warnings","stages","join","STATE_DIR","mkdirSync","recursive","filename","createBuilder","reconcileDialect","exec","getMeta","featureSignature","FEATURES","schemaVersion","console","error","close","clearCache","changedSignatureKeys","isPresetOnlyChange","forcedPresetPaths","size","has","classifyEmbedChange","isFeatureOnlyChange","classifyFeatureToggles","embedIdentityAdopted","setMeta","signatureDiff","ensureSchema","setDerivedBusyTimeout","Number","min","max","invalidate","invalidateFeatures","build","resolvedCfg","createStore","rekeyChunkText","stmt","prepare","get","n"],"mappings":";;;;;;;;;;;QAoMsBA;eAAAA;;QARAC;eAAAA;;;uCA5LE;sBACE;wBACL;uBAEuB;yBACb;wBACJ;wBACF;yBACK;uBACH;4BAES;8BAEwB;0BAC3B;6BACqB;wBACrB;2BACyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8B1E,iGAAiG;AACjG,oFAAoF;AACpF,IAAMC,eAAe;AACrB,IAAMC,WAAWC,IAAAA,gCAAO,EAAC;AAEzB,SAAeC,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;;YAChGC,UACAC,UAIKC,KACFH;;;;oBANHC,WAAWG,IAAAA,4BAAgB,EAACL,IAAIM,OAAO;oBACvCH,WAAWI,KAAKC,GAAG,KAAKN;;;;;;;;;oBAGnB;;wBAAMD,QAAQQ,OAAO,CAACV,QAAQC;;;oBAArC;;wBAAO;;;oBACAI;oBACP,IAAI,GAACH,oBAAAA,QAAQS,QAAQ,cAAhBT,wCAAAA,uBAAAA,SAAmBG,OAAe,MAAMA;oBAC7C,sFAAsF;oBACtF,IAAIR,SAASe,cAAc,EAAEf,SAASgB,OAAO,CAAC;wBAAEC,OAAOb,IAAIa,KAAK;wBAAEd,QAAAA;oBAAO;oBACzE,IAAIQ,KAAKC,GAAG,MAAML,UAAU;wBAC1B,MAAM,IAAIW,oBAAU,CAAC,cAAc,AAAC,8CAAiEf,OAApBC,IAAIa,KAAK,EAAC,YAAmDE,OAAzChB,QAAO,oCAA8D,OAA5BgB,KAAKC,KAAK,CAACd,WAAW,OAAM;oBAC5K;oBACA;;wBAAM,IAAIe,QAAQ,SAACC;mCAAYC,WAAWD,SAASvB;;;;oBAAnD;;;;;;;;;;;;;;;;IAGN;;AAEA,SAAeyB,mBAA2BpB,GAAmB,EAAEC,OAA4B;;YACnFoB,UAEAtB,QAEmB,MAAjBuB,QAAQC,MAGZC,QAGEC,SAMEC,SACAC,UACAC,cAGFC,aAEAC,iBAEAC,gBAYIC,aAIAC,cAGAC,WAIAC,SAkBEC,SASAA,UAIAA,UAcsB,OAAxBC,eAOFC,mBACAC,sBAE+B,OAA7BC,QAAQC,UAAUC,QAKnBtC;;;;oBA5GHiB,WAAWsB,IAAAA,cAAI,EAAC3C,IAAIM,OAAO,EAAEsC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAACxB,UAAU;wBAAEyB,WAAW;oBAAK;oBAChC/C,SAAS4C,IAAAA,cAAI,EAACtB,UAAUpB,QAAQ8C,QAAQ;oBAErB;;wBAAMjD,gBAAgBC,QAAQC,KAAKC;;;oBAAnC,OAAA,eAAjBqB,SAAiB,KAAjBA,QAAQC,OAAS,KAATA;oBAChB,gGAAgG;oBAChG,kGAAkG;oBAC9FC,SAAS;oBACb,+FAA+F;oBAC/F,2CAA2C;oBACrCC,UAAUuB,IAAAA,wBAAa,EAACzB,MAAMvB,KAAKA,IAAIM,OAAO,EAAEL,QAAQgD,gBAAgB;;;;;;;;;oBAE5E;;wBAAM1B,KAAK2B,IAAI,CAAC;;;oBAAhB;oBAIgB;;wBAAMC,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA9BG,UAAU;oBACC;;wBAAMyB,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA/BI,WAAW;oBACXC,eAAewB,IAAAA,yBAAgB,EAACpD,KAAKqD,kBAAQ;yBAQ/C,CAAA,AAAC3B,YAAY,QAAQA,YAAYzB,QAAQqD,aAAa,IAAM3B,aAAa,QAAQA,aAAaC,YAAY,GAA1G;;;;yBAGEF,CAAAA,YAAY,QAAQA,YAAYzB,QAAQqD,aAAa,AAAD,GAApD5B;;;;oBACF6B,QAAQC,KAAK,CAAC;oBACd,kFAAkF;oBAClFhC,SAAS;oBACT;;wBAAMvB,QAAQwD,KAAK,CAACnC;;;oBAApB;oBACAoC,IAAAA,mBAAU,EAAC1D;oBACX;;wBAAOoB,mBAAmBpB,KAAKC;;;oBAE3B+B,cAAc2B,IAAAA,iCAAoB,EAAChC,qBAAAA,sBAAAA,WAAY,IAAIC;oBACzD,oFAAoF;oBACpF,0FAA0F;oBAC1F,2EAA2E;oBACrEK,eAAe2B,IAAAA,iCAAkB,EAAC5B,eAAe6B,IAAAA,gCAAiB,EAAC7D,KAAKA,IAAIM,OAAO,EAAEqB,qBAAAA,sBAAAA,WAAY,IAAIK,eAAe;oBAC1H,yFAAyF;oBACzF,oFAAoF;oBAC9EE,YAAYF,YAAY8B,IAAI,KAAK,KAAK9B,YAAY+B,GAAG,CAAC,WAAWC,IAAAA,iCAAmB,EAACrC,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;oBAC3H,4FAA4F;oBAC5F,6FAA6F;oBAC7F,4DAA4D;oBACtDO,UAAU8B,IAAAA,mCAAmB,EAACjC,eAAekC,IAAAA,sCAAsB,EAACvC,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcI,eAAe;yBACnHA,CAAAA,YAAY8B,IAAI,KAAK,KAAK9B,YAAY+B,GAAG,CAAC,YAAYI,IAAAA,iCAAoB,EAACxC,qBAAAA,sBAAAA,WAAY,IAAIC,aAAY,GAAvGI;;;;oBACF,iFAAiF;oBACjF,sDAAsD;oBACtDuB,QAAQC,KAAK,CAAC;oBACd;;wBAAMY,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSM,CAAAA,cAAc,IAAG,GAAjBA;;;;oBACT,yFAAyF;oBACzF,iEAAiE;oBACjEJ,kBAAkBI;oBAClBqB,QAAQC,KAAK,CAACtB,cAAc,UAAU,6EAA6E;oBACnH;;wBAAMkC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSK,CAAAA,iBAAiB,IAAG,GAApBA;;;;oBACT,kFAAkF;oBAClF,uEAAuE;oBACvE,sFAAsF;oBACtF,qFAAqF;oBACrFJ,cAAcI;oBACRG,UAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,SAAQ;oBAC/C;;wBAAMgC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSO,CAAAA,YAAY,IAAG,GAAfA;;;;oBACT,uFAAuF;oBACvF,wFAAwF;oBACxF,kFAAkF;oBAClF,sDAAsD;oBACtDJ,iBAAiBI;oBACXC,WAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,UAAQ;oBAC/C;;wBAAMgC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;oBAEMQ,WAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,UAAQ;oBAC/CZ,SAAS;oBACT;;wBAAMvB,QAAQwD,KAAK,CAACnC;;;oBAApB;oBACAoC,IAAAA,mBAAU,EAAC1D;oBACX;;wBAAOoB,mBAAmBpB,KAAKC;;;oBAInC;;wBAAMA,QAAQqE,YAAY,CAAChD,QAAQC,MAAMvB;;;oBAAzC;yBAIIC,QAAQsE,qBAAqB,EAA7BtE;;;;oBAC4B;;wBAAMkD,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA5Cc,gBAAgBmC;yBAAQ,QAAA,2BAAA,mBAAA,QAA4C;;oBAC1E;;wBAAMvE,QAAQsE,qBAAqB,CAACjD,QAAQC,MAAMR,KAAK0D,GAAG,CAAC1D,KAAK2D,GAAG,CAAC,OAAO,IAAIrC,gBAAgB;;;oBAA/F;;;yBAMkBP,iBAAAA;;;;oBAAmB;;wBAAML,QAAQkD,UAAU,CAAC7C;;;2BAA1B,AAAC,cAA2CU,MAAM;;;;;;2BAAG;;;oBAArFF;yBACgBP,gBAAAA;;;;oBAAkB;;wBAAMN,QAAQmD,kBAAkB,CAAC7C;;;4BAAlC,AAAC,cAAkDS,MAAM;;;;;;4BAAG;;;oBAA7FD;oBAE+B;;wBAAMd,QAAQoD,KAAK,CAAChD;;;oBAApB,QAAA,eAA7BW,SAA6B,MAA7BA,QAAQC,WAAqB,MAArBA,UAAUC,SAAW,MAAXA;oBAC1B,uIAAuI;oBACvI;;wBAAMjB,QAAQgC,KAAK;;;oBAAnB;oBAEA;;wBAAO;4BAAEnC,QAAAA;4BAAQC,MAAAA;4BAAMvB,KAAAA;4BAAKD,QAAAA;4BAAQyC,QAAQA,SAASF,cAAcC;4BAAeE,UAAAA;4BAAUC,QAAAA;wBAAO;;;oBAC5FtC;oBACP;;wBAAMqB,QAAQgC,KAAK;;;oBAAnB;yBACI,CAACjC,QAAD;;;;oBAAS;;wBAAMvB,QAAQwD,KAAK,CAACnC;;;oBAApB;;;oBACb,MAAMlB;;;;;;;IAEV;;AAIO,SAAeV,gBAAwBM,GAAmB,EAAEC,OAA4B;;YAChB,MAArEqB,QAAQC,MAAWuD,aAAa/E,QAAQyC,QAAQC,UAAUC,QAC5D7B;;;;oBADuE;;wBAAMO,mBAAmBpB,KAAKC;;;oBAA9B,OAAA,eAArEqB,SAAqE,KAArEA,QAAQC,OAA6D,KAA7DA,MAAWuD,cAAkD,KAAvD9E,KAAkBD,SAAqC,KAArCA,QAAQyC,SAA6B,KAA7BA,QAAQC,WAAqB,KAArBA,UAAUC,SAAW,KAAXA;oBAC5D7B,QAAQZ,QAAQ8E,WAAW,CAACzD,QAAQC,MAAMuD;oBAChD,0FAA0F;oBAC1FE,IAAAA,yBAAc,EAACzD,MAAMV;oBACrB;;wBAAO;4BAAEA,OAAAA;4BAAOb,KAAK8E;4BAAa/E,QAAAA;4BAAQyC,QAAAA;4BAAQC,UAAAA;4BAAUC,QAAAA;wBAAO;;;;IACrE;;AAEO,SAAejD,SAASoB,KAAY;;YACnCoE;;;;oBAAO;;wBAAMpE,MAAMqE,OAAO,CAAC;;;oBAA3BD,OAAO;oBACJ;;wBAAMA,KAAKE,GAAG;;;oBAAvB;;wBAAS,cAAoCC,CAAC;;;;IAChD"}

package/dist/cjs/store/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\n// Rows that cannot conflict, at whatever speed this connection offers: the append path where the\n// store has one, else `conflictSql` bound the ordinary way. `conflictSql` stays the caller's own\n// guarded statement, so a store without appendRows behaves exactly as it did before.\nexport async function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  if (conn.appendRows) await conn.appendRows(table, columns, rows);\n  else await conn.runBatch(conflictSql, rows);\n}\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["appendRows","getColumns","getMeta","quoteIdent","recordReconcileDuration","setMeta","conn","table","columns","conflictSql","rows","length","runBatch","name","split","join","stmt","prepare","all","Set","map","r","key","row","get","value","run","ms","prevRaw","prevMax","Number","String"],"mappings":";;;;;;;;;;;QAQsBA;eAAAA;;QAUAC;eAAAA;;QAMAC;eAAAA;;QAVNC;eAAAA;;QA4BMC;eAAAA;;QAZAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAtBf,SAAeL,WAAWM,IAAgB,EAAEC,KAAa,EAAEC,OAAiB,EAAEC,WAAmB,EAAEC,IAAiB;;;;;oBACzH,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;yBACnBL,KAAKN,UAAU,EAAfM;;;;oBAAiB;;wBAAMA,KAAKN,UAAU,CAACO,OAAOC,SAASE;;;oBAAtC;;;;;;oBAChB;;wBAAMJ,KAAKM,QAAQ,CAACH,aAAaC;;;oBAAjC;;;;;;;;IACP;;AAEO,SAASP,WAAWU,IAAY;IACrC,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEO,SAAed,WAAWK,IAAgB;;YACzCU,MACAN;;;;oBADO;;wBAAMJ,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACC;;wBAAMA,KAAKE,GAAG;;;oBAAtBR,OAAQ;oBACd;;wBAAO,IAAIS,IAAIT,KAAKU,GAAG,CAAC,SAACC;mCAAMA,EAAER,IAAI;;;;;IACvC;;AAEO,SAAeX,QAAQI,IAAgB,EAAEgB,GAAW;;YACnDN,MACAO;;;;oBADO;;wBAAMjB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACA;;wBAAMA,KAAKQ,GAAG,CAACF;;;oBAAtBC,MAAO;oBACb;;wBAAOA,MAAMA,IAAIE,KAAK,GAAG;;;;IAC3B;;AAEO,SAAepB,QAAQC,IAAgB,EAAEgB,GAAW,EAAEG,KAAoB;;YAEvET,MAIFA;;;;yBALFS,CAAAA,UAAU,IAAG,GAAbA;;;;oBACW;;wBAAMnB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACb;;wBAAMA,KAAKU,GAAG,CAACJ;;;oBAAf;oBACA;;;;oBAEW;;wBAAMhB,KAAKW,OAAO,CAAC;;;oBAA1BD,QAAO;oBACb;;wBAAMA,MAAKU,GAAG,CAACJ,KAAKG;;;oBAApB;;;;;;IACF;;AAIO,SAAerB,wBAAwBE,IAAgB,EAAEqB,EAAU;;YAClEC,SAGAC;;;;oBAHU;;wBAAM3B,QAAQI,MAAM;;;oBAA9BsB,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;yBAC3CD,CAAAA,KAAKE,OAAM,GAAXF;;;;oBAAc;;wBAAMtB,QAAQC,MAAM,oBAAoByB,OAAOJ;;;oBAA/C;;;;;;;;IACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\n// Rows that cannot conflict, at whatever speed this connection offers: the append path where the\n// store has one, else the caller's own guarded `conflictSql` statement bound the ordinary way.\nexport async function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  if (conn.appendRows) await conn.appendRows(table, columns, rows);\n  else await conn.runBatch(conflictSql, rows);\n}\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["appendRows","getColumns","getMeta","quoteIdent","recordReconcileDuration","setMeta","conn","table","columns","conflictSql","rows","length","runBatch","name","split","join","stmt","prepare","all","Set","map","r","key","row","get","value","run","ms","prevRaw","prevMax","Number","String"],"mappings":";;;;;;;;;;;QAOsBA;eAAAA;;QAUAC;eAAAA;;QAMAC;eAAAA;;QAVNC;eAAAA;;QA4BMC;eAAAA;;QAZAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAtBf,SAAeL,WAAWM,IAAgB,EAAEC,KAAa,EAAEC,OAAiB,EAAEC,WAAmB,EAAEC,IAAiB;;;;;oBACzH,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;yBACnBL,KAAKN,UAAU,EAAfM;;;;oBAAiB;;wBAAMA,KAAKN,UAAU,CAACO,OAAOC,SAASE;;;oBAAtC;;;;;;oBAChB;;wBAAMJ,KAAKM,QAAQ,CAACH,aAAaC;;;oBAAjC;;;;;;;;IACP;;AAEO,SAASP,WAAWU,IAAY;IACrC,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEO,SAAed,WAAWK,IAAgB;;YACzCU,MACAN;;;;oBADO;;wBAAMJ,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACC;;wBAAMA,KAAKE,GAAG;;;oBAAtBR,OAAQ;oBACd;;wBAAO,IAAIS,IAAIT,KAAKU,GAAG,CAAC,SAACC;mCAAMA,EAAER,IAAI;;;;;IACvC;;AAEO,SAAeX,QAAQI,IAAgB,EAAEgB,GAAW;;YACnDN,MACAO;;;;oBADO;;wBAAMjB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACA;;wBAAMA,KAAKQ,GAAG,CAACF;;;oBAAtBC,MAAO;oBACb;;wBAAOA,MAAMA,IAAIE,KAAK,GAAG;;;;IAC3B;;AAEO,SAAepB,QAAQC,IAAgB,EAAEgB,GAAW,EAAEG,KAAoB;;YAEvET,MAIFA;;;;yBALFS,CAAAA,UAAU,IAAG,GAAbA;;;;oBACW;;wBAAMnB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACb;;wBAAMA,KAAKU,GAAG,CAACJ;;;oBAAf;oBACA;;;;oBAEW;;wBAAMhB,KAAKW,OAAO,CAAC;;;oBAA1BD,QAAO;oBACb;;wBAAMA,MAAKU,GAAG,CAACJ,KAAKG;;;oBAApB;;;;;;IACF;;AAIO,SAAerB,wBAAwBE,IAAgB,EAAEqB,EAAU;;YAClEC,SAGAC;;;;oBAHU;;wBAAM3B,QAAQI,MAAM;;;oBAA9BsB,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;yBAC3CD,CAAAA,KAAKE,OAAM,GAAXF;;;;oBAAc;;wBAAMtB,QAAQC,MAAM,oBAAoByB,OAAOJ;;;oBAA/C;;;;;;;;IACpB"}

package/dist/cjs/store/sqlite/lexical.js

@@ -134,9 +134,6 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
-// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
-// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.
-var SNIPPET_BOUND = 16384;
 var WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
 function queryLexical(conn, terms, opts) {
     return _async_to_generator(function() {
@@ -152,7 +149,7 @@ function queryLexical(conn, terms, opts) {
                         2,
                         []
                     ];
-                    sql = "SELECT content.path AS path, CASE WHEN length(content.text) <= ".concat(SNIPPET_BOUND, " THEN snippet(content, 2, '\xab', '\xbb', '…', 10) ELSE NULL END AS hit FROM content ").concat(whereJoin, " WHERE content MATCH ? ").concat(whereCond, " ").concat(scopeCond, " ORDER BY ").concat(WEIGHTED_BM25, " LIMIT ").concat(limit);
+                    sql = "SELECT content.path AS path FROM content ".concat(whereJoin, " WHERE content MATCH ? ").concat(whereCond, " ").concat(scopeCond, " ORDER BY ").concat(WEIGHTED_BM25, " LIMIT ").concat(limit);
                     return [
                         4,
                         conn.prepare(sql)

package/dist/cjs/store/sqlite/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. An\n// unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmentMatch(terms);\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["queryLexical","SNIPPET_BOUND","WEIGHTED_BM25","conn","terms","opts","whereJoin","whereCond","scopeCond","limit","query","sql","stmt","segmentMatch","trim","prepare","all"],"mappings":";;;;+BAUsBA;;;eAAAA;;;yBAVO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG7B,iFAAiF;AACjF,sIAAsI;AACtI,IAAMC,gBAAgB;AACtB,IAAMC,gBAAgB;AAIf,SAAeF,aAAaG,IAAgB,EAAEC,KAAa,EAAEC,IAAyB;;YACnFC,WAAWC,WAAWC,WAAWC,OACnCC,OAIAC,KACAC;;;;oBANEN,YAA2CD,KAA3CC,WAAWC,YAAgCF,KAAhCE,WAAWC,YAAqBH,KAArBG,WAAWC,QAAUJ,KAAVI;oBACnCC,QAAQG,IAAAA,uBAAY,EAACT;oBAC3B,2FAA2F;oBAC3F,sGAAsG;oBACtG,IAAIM,MAAMI,IAAI,OAAO,IAAI;;;;oBACnBH,MAAM,AAAC,kEAAgKL,OAA/FL,eAAc,yFAAoHM,OAAnCD,WAAU,2BAAsCE,OAAbD,WAAU,KAAyBL,OAAtBM,WAAU,cAAmCC,OAAvBP,eAAc,WAAe,OAANO;oBAC7P;;wBAAMN,KAAKY,OAAO,CAACJ;;;oBAA1BC,OAAO;oBACL;;wBAAMA,KAAKI,GAAG,CAACN;;;oBAAvB;;wBAAQ;;;;IACV"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query, scoped by the caller-built SQL fragments. An\n// unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmentMatch(terms);\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["queryLexical","WEIGHTED_BM25","conn","terms","opts","whereJoin","whereCond","scopeCond","limit","query","sql","stmt","segmentMatch","trim","prepare","all"],"mappings":";;;;+BAOsBA;;;eAAAA;;;yBAPO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG7B,IAAMC,gBAAgB;AAIf,SAAeD,aAAaE,IAAgB,EAAEC,KAAa,EAAEC,IAAyB;;YACnFC,WAAWC,WAAWC,WAAWC,OACnCC,OAIAC,KACAC;;;;oBANEN,YAA2CD,KAA3CC,WAAWC,YAAgCF,KAAhCE,WAAWC,YAAqBH,KAArBG,WAAWC,QAAUJ,KAAVI;oBACnCC,QAAQG,IAAAA,uBAAY,EAACT;oBAC3B,2FAA2F;oBAC3F,sGAAsG;oBACtG,IAAIM,MAAMI,IAAI,OAAO,IAAI;;;;oBACnBH,MAAM,AAAC,4CAA8EJ,OAAnCD,WAAU,2BAAsCE,OAAbD,WAAU,KAAyBL,OAAtBM,WAAU,cAAmCC,OAAvBP,eAAc,WAAe,OAANO;oBACxI;;wBAAMN,KAAKY,OAAO,CAACJ;;;oBAA1BC,OAAO;oBACL;;wBAAMA,KAAKI,GAAG,CAACN;;;oBAAvB;;wBAAQ;;;;IACV"}

package/dist/cjs/store/sqlite/store.js

@@ -361,11 +361,8 @@ function _wrap_async_generator(fn) {
     };
 }
 var CAPABILITIES = new Set([
-    'phrases',
-    'snippets',
-    'lexical',
     'vectors',
-    'sql-functions'
+    'segment'
 ]);
 function createStore(db, conn) {
     return {

package/dist/cjs/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QASaA;eAAAA;;QAIGC;eAAAA;;;wBAZW;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB;AAIpH,SAASD,YAAYE,EAAgB,EAAEC,IAAgB;IAC5D,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMc,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;;gBACnC;;QACF;QACAE,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAACvB;;;gBACrB;;YACMwB,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC1B,MAAMyB;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAAChC,MAAM4B,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEf,IAAI;;;wBACtB;;4BAAOgB,IAAAA,uBAAW,EAACnC,MAAMkC,MAAMf;;;gBACjC;;YACMiB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAACrC,MAAMkC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAMxC,GAAGM,OAAO,CAAC,uBAAuBmC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;wBAAAA,OAAO7C,GAAGM,OAAO,CAACD;wBACxBwC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLjC,SAAS,SAATA;2CAAegC,KAAKhC,OAAO;;gCAC3B,2EAA2E;gCAC3EkC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJjD,GAAGiD,KAAK;;;;;YACV;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors', 'segment']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QASaA;eAAAA;;QAIGC;eAAAA;;;wBAZW;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;CAAU;AAI5E,SAASD,YAAYE,EAAgB,EAAEC,IAAgB;IAC5D,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMc,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;;gBACnC;;QACF;QACAE,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAACvB;;;gBACrB;;YACMwB,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC1B,MAAMyB;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAAChC,MAAM4B,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEf,IAAI;;;wBACtB;;4BAAOgB,IAAAA,uBAAW,EAACnC,MAAMkC,MAAMf;;;gBACjC;;YACMiB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAACrC,MAAMkC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAMxC,GAAGM,OAAO,CAAC,uBAAuBmC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;wBAAAA,OAAO7C,GAAGM,OAAO,CAACD;wBACxBwC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLjC,SAAS,SAATA;2CAAegC,KAAKhC,OAAO;;gCAC3B,2EAA2E;gCAC3EkC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJjD,GAAGiD,KAAK;;;;;YACV;;IACF;AACF"}