sensemaking 0.23.0 → 0.24.0

package/dist/cjs/store/sqlite/vectors.js

@@ -57,6 +57,17 @@ function _async_to_generator(fn) {
         });
     };
 }
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
 function _iterable_to_array(iter) {
     if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
         return Array.from(iter);
@@ -92,6 +103,21 @@ function _non_iterable_rest() {
 function _non_iterable_spread() {
     throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
 }
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
 function _sliced_to_array(arr, i) {
     return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
 }
@@ -235,13 +261,13 @@ function writeVectorBatch(conn, rows) {
 }
 function scanCandidates(conn, qv, storeDims, fetch, allowed) {
     return _async_to_generator(function() {
-        var stmt, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, q, dot, d, score, existing;
+        var stmt, rows, queryNorm, d, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, q, dot, norm, length, d1, d2, score, existing;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
                     return [
                         4,
-                        conn.prepare('SELECT "path", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL')
+                        conn.prepare('SELECT "path", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY "path", chunk')
                     ];
                 case 1:
                     stmt = _state.sent();
@@ -251,6 +277,9 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
                     ];
                 case 2:
                     rows = _state.sent();
+                    queryNorm = 0;
+                    for(d = 0; d < Math.min(storeDims, qv.length); d++)queryNorm += qv[d] * qv[d];
+                    queryNorm = Math.sqrt(queryNorm);
                     best = new Map();
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
@@ -258,14 +287,25 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
                             row = _step.value;
                             if (allowed && !allowed.has(row.path)) continue;
                             q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));
+                            // Rounding to int8 costs the stored row its unit norm, so dividing by that norm is what makes
+                            // this a cosine rather than a scaled dot product. `scale` cancels out of the ratio.
                             dot = 0;
-                            for(d = 0; d < q.length; d++)dot += q[d] * qv[d];
-                            score = dot * row.scale;
+                            norm = 0;
+                            length = Math.min(q.length, qv.length);
+                            for(d1 = 0; d1 < length; d1++){
+                                dot += q[d1] * qv[d1];
+                                norm += q[d1] * q[d1];
+                            }
+                            for(d2 = length; d2 < q.length; d2++)norm += q[d2] * q[d2];
+                            score = norm === 0 || queryNorm === 0 ? 0 : dot / (Math.sqrt(norm) * queryNorm);
                             existing = best.get(row.path);
-                            if (!existing || score > existing.score) best.set(row.path, {
-                                score: score,
-                                lines: "L".concat(row.start_line, "-").concat(row.end_line)
-                            });
+                            if (!existing || score > existing.score || score === existing.score && row.chunk < existing.chunk) {
+                                best.set(row.path, {
+                                    score: score,
+                                    chunk: row.chunk,
+                                    lines: "L".concat(row.start_line, "-").concat(row.end_line)
+                                });
+                            }
                         }
                     } catch (err) {
                         _didIteratorError = true;
@@ -283,14 +323,17 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
                     }
                     return [
                         2,
-                        _to_consumable_array(best.entries()).sort(function(a, b) {
-                            return b[1].score - a[1].score;
-                        }).slice(0, fetch).map(function(param) {
-                            var _param = _sliced_to_array(param, 2), path = _param[0], b = _param[1];
+                        _to_consumable_array(best.entries()).map(function(param) {
+                            var _param = _sliced_to_array(param, 2), path = _param[0], value = _param[1];
+                            return _object_spread({
+                                path: path
+                            }, value);
+                        }).sort(_vectorsts.compareVectorScores).slice(0, fetch).map(function(param) {
+                            var path = param.path, lines = param.lines, score = param.score;
                             return {
                                 path: path,
-                                lines: b.lines,
-                                similarity: (0, _vectorsts.asCosine)(b.score)
+                                lines: lines,
+                                similarity: (0, _vectorsts.asCosine)(score)
                             };
                         })
                     ];
@@ -300,13 +343,13 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
 }
 function scanSimilar(conn, path, opts) {
     return _async_to_generator(function() {
-        var targetStmt, targetRows, target, stmt, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, other, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, t, dot, len, d, score, existing;
+        var targetStmt, targetRows, target, stmt, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, other, otherNorm, d, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, t, dot, len, d1, score, existing;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
                     return [
                         4,
-                        conn.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk')
+                        conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk')
                     ];
                 case 1:
                     targetStmt = _state.sent();
@@ -320,15 +363,19 @@ function scanSimilar(conn, path, opts) {
                         2,
                         []
                     ];
+                    // Each side's norm divides its own dot product, so both are taken once here rather than per pair.
                     target = (0, _vectorsts.sampleEvenly)(targetRows).map(function(row) {
+                        var v = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+                        var norm = 0;
+                        for(var d = 0; d < v.length; d++)norm += v[d] * v[d];
                         return {
-                            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
-                            scale: row.scale
+                            v: v,
+                            norm: Math.sqrt(norm)
                         };
                     });
                     return [
                         4,
-                        conn.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL')
+                        conn.prepare('SELECT "path", vector FROM embeddings WHERE vector IS NOT NULL')
                     ];
                 case 3:
                     stmt = _state.sent();
@@ -345,14 +392,17 @@ function scanSimilar(conn, path, opts) {
                             row = _step.value;
                             if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
                             other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+                            otherNorm = 0;
+                            for(d = 0; d < other.length; d++)otherNorm += other[d] * other[d];
+                            otherNorm = Math.sqrt(otherNorm);
                             _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
                             try {
                                 for(_iterator1 = target[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
                                     t = _step1.value;
                                     dot = 0;
                                     len = Math.min(t.v.length, other.length);
-                                    for(d = 0; d < len; d++)dot += t.v[d] * other[d];
-                                    score = dot * t.scale * row.scale;
+                                    for(d1 = 0; d1 < len; d1++)dot += t.v[d1] * other[d1];
+                                    score = t.norm === 0 || otherNorm === 0 ? 0 : dot / (t.norm * otherNorm);
                                     existing = best.get(row.path);
                                     if (existing === undefined || score > existing) best.set(row.path, score);
                                 }
@@ -387,10 +437,14 @@ function scanSimilar(conn, path, opts) {
                     }
                     return [
                         2,
-                        _to_consumable_array(best.entries()).sort(function(a, b) {
-                            return b[1] - a[1];
-                        }).slice(0, opts.k).map(function(param) {
+                        _to_consumable_array(best.entries()).map(function(param) {
                             var _param = _sliced_to_array(param, 2), p = _param[0], score = _param[1];
+                            return {
+                                path: p,
+                                score: score
+                            };
+                        }).sort(_vectorsts.compareVectorScores).slice(0, opts.k).map(function(param) {
+                            var p = param.path, score = param.score;
                             return {
                                 path: p,
                                 similarity: (0, _vectorsts.asCosine)(score)

package/dist/cjs/store/sqlite/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const stmt = await conn.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  const targetStmt = await conn.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const target = sampleEvenly(targetRows).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const stmt = await conn.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","conn","rows","length","runBatch","map","row","scale","vector","path","chunk","qv","storeDims","fetch","allowed","stmt","best","q","dot","d","score","existing","prepare","all","Map","has","Int8Array","buffer","byteOffset","Math","min","byteLength","get","set","lines","start_line","end_line","entries","sort","a","b","slice","similarity","asCosine","opts","targetStmt","targetRows","target","other","t","len","sampleEvenly","v","exclude","undefined","k","p"],"mappings":";;;;;;;;;;;QAesBA;eAAAA;;QA4BAC;eAAAA;;QAtCAC;eAAAA;;;yBAJiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIhC,SAAeA,iBAAiBC,IAAgB,EAAEC,IAAsB;;;;;oBAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;oBACvB;;wBAAMF,KAAKG,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,SAACC;mCAAQ;gCAACA,IAAIC,KAAK;gCAAED,IAAIE,MAAM;gCAAEF,IAAIG,IAAI;gCAAEH,IAAII,KAAK;6BAAC;;;;oBAFhE;;;;;;IAIF;;AAIO,SAAeZ,eAAeG,IAAgB,EAAEU,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;;YACxHC,MACAb,MAQAc,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,KAEHW,GACFC,KACKC,GACHC,OACAC;;;;oBAhBK;;wBAAMpB,KAAKqB,OAAO,CAAC;;;oBAA1BP,OAAO;oBACC;;wBAAMA,KAAKQ,GAAG;;;oBAAtBrB,OAAQ;oBAQRc,OAAO,IAAIQ;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAatB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIQ,WAAW,CAACA,QAAQW,GAAG,CAACnB,IAAIG,IAAI,GAAG;4BACjCQ,IAAI,IAAIS,UAAUpB,IAAIE,MAAM,CAACmB,MAAM,EAAErB,IAAIE,MAAM,CAACoB,UAAU,EAAEC,KAAKC,GAAG,CAAClB,WAAWN,IAAIE,MAAM,CAACuB,UAAU;4BACvGb,MAAM;4BACV,IAASC,IAAI,GAAGA,IAAIF,EAAEd,MAAM,EAAEgB,IAAKD,OAAOD,CAAC,CAACE,EAAE,GAAGR,EAAE,CAACQ,EAAE;4BAChDC,QAAQF,MAAMZ,IAAIC,KAAK;4BACvBc,WAAWL,KAAKgB,GAAG,CAAC1B,IAAIG,IAAI;4BAClC,IAAI,CAACY,YAAYD,QAAQC,SAASD,KAAK,EAAEJ,KAAKiB,GAAG,CAAC3B,IAAIG,IAAI,EAAE;gCAAEW,OAAAA;gCAAOc,OAAO,AAAC,IAAqB5B,OAAlBA,IAAI6B,UAAU,EAAC,KAAgB,OAAb7B,IAAI8B,QAAQ;4BAAG;wBACnH;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBASL;;wBAAQ,qBAAGpB,KAAKqB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACpB,KAAK,GAAGmB,CAAC,CAAC,EAAE,CAACnB,KAAK;2BACtCqB,KAAK,CAAC,GAAG5B,OACTR,GAAG,CAAC;qEAAEI,kBAAM+B;mCAAQ;gCAAE/B,MAAAA;gCAAMyB,OAAOM,EAAEN,KAAK;gCAAEQ,YAAYC,IAAAA,mBAAQ,EAACH,EAAEpB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAAerB,YAAYE,IAAgB,EAAEQ,IAAY,EAAEmC,IAAgE;;YAC1HC,YACAC,YAEAC,QAEAhC,MACAb,MACAc,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,KAEH0C,OACD,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,GACL/B,KACEgC,KACG/B,GACHC,OACAC;;;;oBAhBS;;wBAAMpB,KAAKqB,OAAO,CAAC;;;oBAAhCuB,aAAa;oBACC;;wBAAMA,WAAWtB,GAAG,CAACd;;;oBAAnCqC,aAAc;oBACpB,IAAIA,WAAW3C,MAAM,KAAK,GAAG;;;;oBACvB4C,SAASI,IAAAA,uBAAY,EAACL,YAAYzC,GAAG,CAAC,SAACC;+BAAS;4BAAE8C,GAAG,IAAI1B,UAAUpB,IAAIE,MAAM,CAACmB,MAAM,EAAErB,IAAIE,MAAM,CAACoB,UAAU,EAAEtB,IAAIE,MAAM,CAACuB,UAAU;4BAAGxB,OAAOD,IAAIC,KAAK;wBAAC;;oBAE/I;;wBAAMN,KAAKqB,OAAO,CAAC;;;oBAA1BP,OAAO;oBACC;;wBAAMA,KAAKQ,GAAG;;;oBAAtBrB,OAAQ;oBACRc,OAAO,IAAIQ;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAatB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQmC,KAAKS,OAAO,CAAC5B,GAAG,CAACnB,IAAIG,IAAI,KAAMmC,KAAK9B,OAAO,IAAI,CAAC8B,KAAK9B,OAAO,CAACW,GAAG,CAACnB,IAAIG,IAAI,GAAI;4BAChGuC,QAAQ,IAAItB,UAAUpB,IAAIE,MAAM,CAACmB,MAAM,EAAErB,IAAIE,MAAM,CAACoB,UAAU,EAAEtB,IAAIE,MAAM,CAACuB,UAAU;4BACtF,mCAAA,4BAAA;;gCAAL,IAAK,aAAWgB,6BAAX,8BAAA,SAAA,0BAAA,kCAAmB;oCAAbE,IAAN;oCACC/B,MAAM;oCACJgC,MAAMrB,KAAKC,GAAG,CAACmB,EAAEG,CAAC,CAACjD,MAAM,EAAE6C,MAAM7C,MAAM;oCAC7C,IAASgB,IAAI,GAAGA,IAAI+B,KAAK/B,IAAKD,OAAO+B,EAAEG,CAAC,CAACjC,EAAE,GAAG6B,KAAK,CAAC7B,EAAE;oCAChDC,QAAQF,MAAM+B,EAAE1C,KAAK,GAAGD,IAAIC,KAAK;oCACjCc,WAAWL,KAAKgB,GAAG,CAAC1B,IAAIG,IAAI;oCAClC,IAAIY,aAAaiC,aAAalC,QAAQC,UAAUL,KAAKiB,GAAG,CAAC3B,IAAIG,IAAI,EAAEW;gCACrE;;gCAPK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAQP;;wBAXK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAaL;;wBAAQ,qBAAGJ,KAAKqB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;2BAC1BE,KAAK,CAAC,GAAGG,KAAKW,CAAC,EACflD,GAAG,CAAC;qEAAEmD,eAAGpC;mCAAY;gCAAEX,MAAM+C;gCAAGd,YAAYC,IAAAA,mBAAQ,EAACvB;4BAAO;;;;;IACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, compareVectorScores, sampleEvenly } from '../vectors.ts';\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY \"path\", chunk');\n  const rows = (await stmt.all()) as Array<{\n    path: string;\n    chunk: number;\n    start_line: number;\n    end_line: number;\n    vector: Uint8Array;\n  }>;\n\n  let queryNorm = 0;\n  for (let d = 0; d < Math.min(storeDims, qv.length); d++) queryNorm += qv[d] * qv[d];\n  queryNorm = Math.sqrt(queryNorm);\n\n  const best = new Map<string, { score: number; chunk: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    // Rounding to int8 costs the stored row its unit norm, so dividing by that norm is what makes\n    // this a cosine rather than a scaled dot product. `scale` cancels out of the ratio.\n    let dot = 0;\n    let norm = 0;\n    const length = Math.min(q.length, qv.length);\n    for (let d = 0; d < length; d++) {\n      dot += q[d] * qv[d];\n      norm += q[d] * q[d];\n    }\n    for (let d = length; d < q.length; d++) norm += q[d] * q[d];\n    const score = norm === 0 || queryNorm === 0 ? 0 : dot / (Math.sqrt(norm) * queryNorm);\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score || (score === existing.score && row.chunk < existing.chunk)) {\n      best.set(row.path, { score, chunk: row.chunk, lines: `L${row.start_line}-${row.end_line}` });\n    }\n  }\n  return [...best.entries()]\n    .map(([path, value]) => ({ path, ...value }))\n    .sort(compareVectorScores)\n    .slice(0, fetch)\n    .map(({ path, lines, score }) => ({ path, lines, similarity: asCosine(score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  // Each side's norm divides its own dot product, so both are taken once here rather than per pair.\n  const target = sampleEvenly(targetRows).map((row) => {\n    const v = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    let norm = 0;\n    for (let d = 0; d < v.length; d++) norm += v[d] * v[d];\n    return { v, norm: Math.sqrt(norm) };\n  });\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    let otherNorm = 0;\n    for (let d = 0; d < other.length; d++) otherNorm += other[d] * other[d];\n    otherNorm = Math.sqrt(otherNorm);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = t.norm === 0 || otherNorm === 0 ? 0 : dot / (t.norm * otherNorm);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .map(([p, score]) => ({ path: p, score }))\n    .sort(compareVectorScores)\n    .slice(0, opts.k)\n    .map(({ path: p, score }) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","conn","rows","length","runBatch","map","row","scale","vector","path","chunk","qv","storeDims","fetch","allowed","stmt","queryNorm","d","best","q","dot","norm","score","existing","prepare","all","Math","min","sqrt","Map","has","Int8Array","buffer","byteOffset","byteLength","get","set","lines","start_line","end_line","entries","value","sort","compareVectorScores","slice","similarity","asCosine","opts","targetStmt","targetRows","target","other","otherNorm","t","len","sampleEvenly","v","exclude","undefined","p","k"],"mappings":";;;;;;;;;;;QAesBA;eAAAA;;QA2CAC;eAAAA;;QArDAC;eAAAA;;;yBAJsC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIrD,SAAeA,iBAAiBC,IAAgB,EAAEC,IAAsB;;;;;oBAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;oBACvB;;wBAAMF,KAAKG,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,SAACC;mCAAQ;gCAACA,IAAIC,KAAK;gCAAED,IAAIE,MAAM;gCAAEF,IAAIG,IAAI;gCAAEH,IAAII,KAAK;6BAAC;;;;oBAFhE;;;;;;IAIF;;AAIO,SAAeZ,eAAeG,IAAgB,EAAEU,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;;YACxHC,MACAb,MAQFc,WACKC,GAGHC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMZ,KAEHa,GAGFC,KACAC,MACElB,QACGc,IAIAA,IACHK,OACAC;;;;oBA5BK;;wBAAMtB,KAAKuB,OAAO,CAAC;;;oBAA1BT,OAAO;oBACC;;wBAAMA,KAAKU,GAAG;;;oBAAtBvB,OAAQ;oBAQVc,YAAY;oBAChB,IAASC,IAAI,GAAGA,IAAIS,KAAKC,GAAG,CAACf,WAAWD,GAAGR,MAAM,GAAGc,IAAKD,aAAaL,EAAE,CAACM,EAAE,GAAGN,EAAE,CAACM,EAAE;oBACnFD,YAAYU,KAAKE,IAAI,CAACZ;oBAEhBE,OAAO,IAAIW;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAa3B,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIQ,WAAW,CAACA,QAAQgB,GAAG,CAACxB,IAAIG,IAAI,GAAG;4BACjCU,IAAI,IAAIY,UAAUzB,IAAIE,MAAM,CAACwB,MAAM,EAAE1B,IAAIE,MAAM,CAACyB,UAAU,EAAEP,KAAKC,GAAG,CAACf,WAAWN,IAAIE,MAAM,CAAC0B,UAAU;4BAC3G,8FAA8F;4BAC9F,oFAAoF;4BAChFd,MAAM;4BACNC,OAAO;4BACLlB,SAASuB,KAAKC,GAAG,CAACR,EAAEhB,MAAM,EAAEQ,GAAGR,MAAM;4BAC3C,IAASc,KAAI,GAAGA,KAAId,QAAQc,KAAK;gCAC/BG,OAAOD,CAAC,CAACF,GAAE,GAAGN,EAAE,CAACM,GAAE;gCACnBI,QAAQF,CAAC,CAACF,GAAE,GAAGE,CAAC,CAACF,GAAE;4BACrB;4BACA,IAASA,KAAId,QAAQc,KAAIE,EAAEhB,MAAM,EAAEc,KAAKI,QAAQF,CAAC,CAACF,GAAE,GAAGE,CAAC,CAACF,GAAE;4BACrDK,QAAQD,SAAS,KAAKL,cAAc,IAAI,IAAII,MAAOM,CAAAA,KAAKE,IAAI,CAACP,QAAQL,SAAQ;4BAC7EO,WAAWL,KAAKiB,GAAG,CAAC7B,IAAIG,IAAI;4BAClC,IAAI,CAACc,YAAYD,QAAQC,SAASD,KAAK,IAAKA,UAAUC,SAASD,KAAK,IAAIhB,IAAII,KAAK,GAAGa,SAASb,KAAK,EAAG;gCACnGQ,KAAKkB,GAAG,CAAC9B,IAAIG,IAAI,EAAE;oCAAEa,OAAAA;oCAAOZ,OAAOJ,IAAII,KAAK;oCAAE2B,OAAO,AAAC,IAAqB/B,OAAlBA,IAAIgC,UAAU,EAAC,KAAgB,OAAbhC,IAAIiC,QAAQ;gCAAG;4BAC5F;wBACF;;wBAlBK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAmBL;;wBAAQ,qBAAGrB,KAAKsB,OAAO,IACpBnC,GAAG,CAAC;qEAAEI,kBAAMgC;mCAAY;gCAAEhC,MAAAA;+BAASgC;2BACnCC,IAAI,CAACC,8BAAmB,EACxBC,KAAK,CAAC,GAAG/B,OACTR,GAAG,CAAC;gCAAGI,aAAAA,MAAM4B,cAAAA,OAAOf,cAAAA;mCAAa;gCAAEb,MAAAA;gCAAM4B,OAAAA;gCAAOQ,YAAYC,IAAAA,mBAAQ,EAACxB;4BAAO;;;;;IACjF;;AAIO,SAAevB,YAAYE,IAAgB,EAAEQ,IAAY,EAAEsC,IAAgE;;YAC1HC,YACAC,YAGAC,QAOAnC,MACAb,MACAgB,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMZ,KAEH6C,OACFC,WACKnC,GAEJ,4BAAA,oBAAA,iBAAA,YAAA,QAAMoC,GACLjC,KACEkC,KACGrC,IACHK,OACAC;;;;oBAzBS;;wBAAMtB,KAAKuB,OAAO,CAAC;;;oBAAhCwB,aAAa;oBACC;;wBAAMA,WAAWvB,GAAG,CAAChB;;;oBAAnCwC,aAAc;oBACpB,IAAIA,WAAW9C,MAAM,KAAK,GAAG;;;;oBAC7B,kGAAkG;oBAC5F+C,SAASK,IAAAA,uBAAY,EAACN,YAAY5C,GAAG,CAAC,SAACC;wBAC3C,IAAMkD,IAAI,IAAIzB,UAAUzB,IAAIE,MAAM,CAACwB,MAAM,EAAE1B,IAAIE,MAAM,CAACyB,UAAU,EAAE3B,IAAIE,MAAM,CAAC0B,UAAU;wBACvF,IAAIb,OAAO;wBACX,IAAK,IAAIJ,IAAI,GAAGA,IAAIuC,EAAErD,MAAM,EAAEc,IAAKI,QAAQmC,CAAC,CAACvC,EAAE,GAAGuC,CAAC,CAACvC,EAAE;wBACtD,OAAO;4BAAEuC,GAAAA;4BAAGnC,MAAMK,KAAKE,IAAI,CAACP;wBAAM;oBACpC;oBAEa;;wBAAMpB,KAAKuB,OAAO,CAAC;;;oBAA1BT,OAAO;oBACC;;wBAAMA,KAAKU,GAAG;;;oBAAtBvB,OAAQ;oBACRgB,OAAO,IAAIW;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAa3B,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQsC,KAAKU,OAAO,CAAC3B,GAAG,CAACxB,IAAIG,IAAI,KAAMsC,KAAKjC,OAAO,IAAI,CAACiC,KAAKjC,OAAO,CAACgB,GAAG,CAACxB,IAAIG,IAAI,GAAI;4BAChG0C,QAAQ,IAAIpB,UAAUzB,IAAIE,MAAM,CAACwB,MAAM,EAAE1B,IAAIE,MAAM,CAACyB,UAAU,EAAE3B,IAAIE,MAAM,CAAC0B,UAAU;4BACvFkB,YAAY;4BAChB,IAASnC,IAAI,GAAGA,IAAIkC,MAAMhD,MAAM,EAAEc,IAAKmC,aAAaD,KAAK,CAAClC,EAAE,GAAGkC,KAAK,CAAClC,EAAE;4BACvEmC,YAAY1B,KAAKE,IAAI,CAACwB;4BACjB,mCAAA,4BAAA;;gCAAL,IAAK,aAAWF,6BAAX,8BAAA,SAAA,0BAAA,kCAAmB;oCAAbG,IAAN;oCACCjC,MAAM;oCACJkC,MAAM5B,KAAKC,GAAG,CAAC0B,EAAEG,CAAC,CAACrD,MAAM,EAAEgD,MAAMhD,MAAM;oCAC7C,IAASc,KAAI,GAAGA,KAAIqC,KAAKrC,KAAKG,OAAOiC,EAAEG,CAAC,CAACvC,GAAE,GAAGkC,KAAK,CAAClC,GAAE;oCAChDK,QAAQ+B,EAAEhC,IAAI,KAAK,KAAK+B,cAAc,IAAI,IAAIhC,MAAOiC,CAAAA,EAAEhC,IAAI,GAAG+B,SAAQ;oCACtE7B,WAAWL,KAAKiB,GAAG,CAAC7B,IAAIG,IAAI;oCAClC,IAAIc,aAAamC,aAAapC,QAAQC,UAAUL,KAAKkB,GAAG,CAAC9B,IAAIG,IAAI,EAAEa;gCACrE;;gCAPK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAQP;;wBAdK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAgBL;;wBAAQ,qBAAGJ,KAAKsB,OAAO,IACpBnC,GAAG,CAAC;qEAAEsD,eAAGrC;mCAAY;gCAAEb,MAAMkD;gCAAGrC,OAAAA;4BAAM;2BACtCoB,IAAI,CAACC,8BAAmB,EACxBC,KAAK,CAAC,GAAGG,KAAKa,CAAC,EACfvD,GAAG,CAAC;gCAAGI,AAAMkD,UAANlD,MAASa,cAAAA;mCAAa;gCAAEb,MAAMkD;gCAAGd,YAAYC,IAAAA,mBAAQ,EAACxB;4BAAO;;;;;IACzE"}

package/dist/cjs/store/turso/connection.d.cts

@@ -1,4 +1,7 @@
 import type { Database } from '@tursodatabase/database';
 import type { Connection } from '../types.js';
 export declare function checkpointWal(db: Database): Promise<void>;
+export declare function cacheFilePath(conn: Connection): Promise<string | null>;
+export declare function fileSize(path: string): Promise<number | null>;
+export declare function reclaimSpace(path: string): Promise<void>;
 export declare function createConnection(db: Database): Connection;

package/dist/cjs/store/turso/connection.d.ts

@@ -1,4 +1,7 @@
 import type { Database } from '@tursodatabase/database';
 import type { Connection } from '../types.js';
 export declare function checkpointWal(db: Database): Promise<void>;
+export declare function cacheFilePath(conn: Connection): Promise<string | null>;
+export declare function fileSize(path: string): Promise<number | null>;
+export declare function reclaimSpace(path: string): Promise<void>;
 export declare function createConnection(db: Database): Connection;