sensemaking 0.23.1 → 0.24.1

package/dist/cjs/store/sqlite/vectors.js

@@ -57,6 +57,17 @@ function _async_to_generator(fn) {
         });
     };
 }
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
 function _iterable_to_array(iter) {
     if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
         return Array.from(iter);
@@ -92,6 +103,21 @@ function _non_iterable_rest() {
 function _non_iterable_spread() {
     throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
 }
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
 function _sliced_to_array(arr, i) {
     return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
 }
@@ -235,13 +261,13 @@ function writeVectorBatch(conn, rows) {
 }
 function scanCandidates(conn, qv, storeDims, fetch, allowed) {
     return _async_to_generator(function() {
-        var stmt, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, q, dot, d, score, existing;
+        var stmt, rows, queryNorm, d, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, q, dot, norm, length, d1, d2, score, existing;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
                     return [
                         4,
-                        conn.prepare('SELECT "path", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL')
+                        conn.prepare('SELECT "path", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY "path", chunk')
                     ];
                 case 1:
                     stmt = _state.sent();
@@ -251,6 +277,9 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
                     ];
                 case 2:
                     rows = _state.sent();
+                    queryNorm = 0;
+                    for(d = 0; d < Math.min(storeDims, qv.length); d++)queryNorm += qv[d] * qv[d];
+                    queryNorm = Math.sqrt(queryNorm);
                     best = new Map();
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
@@ -258,14 +287,25 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
                             row = _step.value;
                             if (allowed && !allowed.has(row.path)) continue;
                             q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));
+                            // Rounding to int8 costs the stored row its unit norm, so dividing by that norm is what makes
+                            // this a cosine rather than a scaled dot product. `scale` cancels out of the ratio.
                             dot = 0;
-                            for(d = 0; d < q.length; d++)dot += q[d] * qv[d];
-                            score = dot * row.scale;
+                            norm = 0;
+                            length = Math.min(q.length, qv.length);
+                            for(d1 = 0; d1 < length; d1++){
+                                dot += q[d1] * qv[d1];
+                                norm += q[d1] * q[d1];
+                            }
+                            for(d2 = length; d2 < q.length; d2++)norm += q[d2] * q[d2];
+                            score = norm === 0 || queryNorm === 0 ? 0 : dot / (Math.sqrt(norm) * queryNorm);
                             existing = best.get(row.path);
-                            if (!existing || score > existing.score) best.set(row.path, {
-                                score: score,
-                                lines: "L".concat(row.start_line, "-").concat(row.end_line)
-                            });
+                            if (!existing || score > existing.score || score === existing.score && row.chunk < existing.chunk) {
+                                best.set(row.path, {
+                                    score: score,
+                                    chunk: row.chunk,
+                                    lines: "L".concat(row.start_line, "-").concat(row.end_line)
+                                });
+                            }
                         }
                     } catch (err) {
                         _didIteratorError = true;
@@ -283,14 +323,17 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
                     }
                     return [
                         2,
-                        _to_consumable_array(best.entries()).sort(function(a, b) {
-                            return b[1].score - a[1].score;
-                        }).slice(0, fetch).map(function(param) {
-                            var _param = _sliced_to_array(param, 2), path = _param[0], b = _param[1];
+                        _to_consumable_array(best.entries()).map(function(param) {
+                            var _param = _sliced_to_array(param, 2), path = _param[0], value = _param[1];
+                            return _object_spread({
+                                path: path
+                            }, value);
+                        }).sort(_vectorsts.compareVectorScores).slice(0, fetch).map(function(param) {
+                            var path = param.path, lines = param.lines, score = param.score;
                             return {
                                 path: path,
-                                lines: b.lines,
-                                similarity: (0, _vectorsts.asCosine)(b.score)
+                                lines: lines,
+                                similarity: (0, _vectorsts.asCosine)(score)
                             };
                         })
                     ];
@@ -300,13 +343,13 @@ function scanCandidates(conn, qv, storeDims, fetch, allowed) {
 }
 function scanSimilar(conn, path, opts) {
     return _async_to_generator(function() {
-        var targetStmt, targetRows, target, stmt, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, other, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, t, dot, len, d, score, existing;
+        var targetStmt, targetRows, target, stmt, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, other, otherNorm, d, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, t, dot, len, d1, score, existing;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
                     return [
                         4,
-                        conn.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk')
+                        conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk')
                     ];
                 case 1:
                     targetStmt = _state.sent();
@@ -320,15 +363,19 @@ function scanSimilar(conn, path, opts) {
                         2,
                         []
                     ];
+                    // Each side's norm divides its own dot product, so both are taken once here rather than per pair.
                     target = (0, _vectorsts.sampleEvenly)(targetRows).map(function(row) {
+                        var v = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+                        var norm = 0;
+                        for(var d = 0; d < v.length; d++)norm += v[d] * v[d];
                         return {
-                            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
-                            scale: row.scale
+                            v: v,
+                            norm: Math.sqrt(norm)
                         };
                     });
                     return [
                         4,
-                        conn.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL')
+                        conn.prepare('SELECT "path", vector FROM embeddings WHERE vector IS NOT NULL')
                     ];
                 case 3:
                     stmt = _state.sent();
@@ -345,14 +392,17 @@ function scanSimilar(conn, path, opts) {
                             row = _step.value;
                             if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
                             other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+                            otherNorm = 0;
+                            for(d = 0; d < other.length; d++)otherNorm += other[d] * other[d];
+                            otherNorm = Math.sqrt(otherNorm);
                             _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
                             try {
                                 for(_iterator1 = target[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
                                     t = _step1.value;
                                     dot = 0;
                                     len = Math.min(t.v.length, other.length);
-                                    for(d = 0; d < len; d++)dot += t.v[d] * other[d];
-                                    score = dot * t.scale * row.scale;
+                                    for(d1 = 0; d1 < len; d1++)dot += t.v[d1] * other[d1];
+                                    score = t.norm === 0 || otherNorm === 0 ? 0 : dot / (t.norm * otherNorm);
                                     existing = best.get(row.path);
                                     if (existing === undefined || score > existing) best.set(row.path, score);
                                 }
@@ -387,10 +437,14 @@ function scanSimilar(conn, path, opts) {
                     }
                     return [
                         2,
-                        _to_consumable_array(best.entries()).sort(function(a, b) {
-                            return b[1] - a[1];
-                        }).slice(0, opts.k).map(function(param) {
+                        _to_consumable_array(best.entries()).map(function(param) {
                             var _param = _sliced_to_array(param, 2), p = _param[0], score = _param[1];
+                            return {
+                                path: p,
+                                score: score
+                            };
+                        }).sort(_vectorsts.compareVectorScores).slice(0, opts.k).map(function(param) {
+                            var p = param.path, score = param.score;
                             return {
                                 path: p,
                                 similarity: (0, _vectorsts.asCosine)(score)

package/dist/cjs/store/sqlite/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const stmt = await conn.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  const targetStmt = await conn.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const target = sampleEvenly(targetRows).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const stmt = await conn.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","conn","rows","length","runBatch","map","row","scale","vector","path","chunk","qv","storeDims","fetch","allowed","stmt","best","q","dot","d","score","existing","prepare","all","Map","has","Int8Array","buffer","byteOffset","Math","min","byteLength","get","set","lines","start_line","end_line","entries","sort","a","b","slice","similarity","asCosine","opts","targetStmt","targetRows","target","other","t","len","sampleEvenly","v","exclude","undefined","k","p"],"mappings":";;;;;;;;;;;QAesBA;eAAAA;;QA4BAC;eAAAA;;QAtCAC;eAAAA;;;yBAJiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIhC,SAAeA,iBAAiBC,IAAgB,EAAEC,IAAsB;;;;;oBAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;oBACvB;;wBAAMF,KAAKG,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,SAACC;mCAAQ;gCAACA,IAAIC,KAAK;gCAAED,IAAIE,MAAM;gCAAEF,IAAIG,IAAI;gCAAEH,IAAII,KAAK;6BAAC;;;;oBAFhE;;;;;;IAIF;;AAIO,SAAeZ,eAAeG,IAAgB,EAAEU,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;;YACxHC,MACAb,MAQAc,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,KAEHW,GACFC,KACKC,GACHC,OACAC;;;;oBAhBK;;wBAAMpB,KAAKqB,OAAO,CAAC;;;oBAA1BP,OAAO;oBACC;;wBAAMA,KAAKQ,GAAG;;;oBAAtBrB,OAAQ;oBAQRc,OAAO,IAAIQ;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAatB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIQ,WAAW,CAACA,QAAQW,GAAG,CAACnB,IAAIG,IAAI,GAAG;4BACjCQ,IAAI,IAAIS,UAAUpB,IAAIE,MAAM,CAACmB,MAAM,EAAErB,IAAIE,MAAM,CAACoB,UAAU,EAAEC,KAAKC,GAAG,CAAClB,WAAWN,IAAIE,MAAM,CAACuB,UAAU;4BACvGb,MAAM;4BACV,IAASC,IAAI,GAAGA,IAAIF,EAAEd,MAAM,EAAEgB,IAAKD,OAAOD,CAAC,CAACE,EAAE,GAAGR,EAAE,CAACQ,EAAE;4BAChDC,QAAQF,MAAMZ,IAAIC,KAAK;4BACvBc,WAAWL,KAAKgB,GAAG,CAAC1B,IAAIG,IAAI;4BAClC,IAAI,CAACY,YAAYD,QAAQC,SAASD,KAAK,EAAEJ,KAAKiB,GAAG,CAAC3B,IAAIG,IAAI,EAAE;gCAAEW,OAAAA;gCAAOc,OAAO,AAAC,IAAqB5B,OAAlBA,IAAI6B,UAAU,EAAC,KAAgB,OAAb7B,IAAI8B,QAAQ;4BAAG;wBACnH;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBASL;;wBAAQ,qBAAGpB,KAAKqB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACpB,KAAK,GAAGmB,CAAC,CAAC,EAAE,CAACnB,KAAK;2BACtCqB,KAAK,CAAC,GAAG5B,OACTR,GAAG,CAAC;qEAAEI,kBAAM+B;mCAAQ;gCAAE/B,MAAAA;gCAAMyB,OAAOM,EAAEN,KAAK;gCAAEQ,YAAYC,IAAAA,mBAAQ,EAACH,EAAEpB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAAerB,YAAYE,IAAgB,EAAEQ,IAAY,EAAEmC,IAAgE;;YAC1HC,YACAC,YAEAC,QAEAhC,MACAb,MACAc,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,KAEH0C,OACD,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,GACL/B,KACEgC,KACG/B,GACHC,OACAC;;;;oBAhBS;;wBAAMpB,KAAKqB,OAAO,CAAC;;;oBAAhCuB,aAAa;oBACC;;wBAAMA,WAAWtB,GAAG,CAACd;;;oBAAnCqC,aAAc;oBACpB,IAAIA,WAAW3C,MAAM,KAAK,GAAG;;;;oBACvB4C,SAASI,IAAAA,uBAAY,EAACL,YAAYzC,GAAG,CAAC,SAACC;+BAAS;4BAAE8C,GAAG,IAAI1B,UAAUpB,IAAIE,MAAM,CAACmB,MAAM,EAAErB,IAAIE,MAAM,CAACoB,UAAU,EAAEtB,IAAIE,MAAM,CAACuB,UAAU;4BAAGxB,OAAOD,IAAIC,KAAK;wBAAC;;oBAE/I;;wBAAMN,KAAKqB,OAAO,CAAC;;;oBAA1BP,OAAO;oBACC;;wBAAMA,KAAKQ,GAAG;;;oBAAtBrB,OAAQ;oBACRc,OAAO,IAAIQ;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAatB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQmC,KAAKS,OAAO,CAAC5B,GAAG,CAACnB,IAAIG,IAAI,KAAMmC,KAAK9B,OAAO,IAAI,CAAC8B,KAAK9B,OAAO,CAACW,GAAG,CAACnB,IAAIG,IAAI,GAAI;4BAChGuC,QAAQ,IAAItB,UAAUpB,IAAIE,MAAM,CAACmB,MAAM,EAAErB,IAAIE,MAAM,CAACoB,UAAU,EAAEtB,IAAIE,MAAM,CAACuB,UAAU;4BACtF,mCAAA,4BAAA;;gCAAL,IAAK,aAAWgB,6BAAX,8BAAA,SAAA,0BAAA,kCAAmB;oCAAbE,IAAN;oCACC/B,MAAM;oCACJgC,MAAMrB,KAAKC,GAAG,CAACmB,EAAEG,CAAC,CAACjD,MAAM,EAAE6C,MAAM7C,MAAM;oCAC7C,IAASgB,IAAI,GAAGA,IAAI+B,KAAK/B,IAAKD,OAAO+B,EAAEG,CAAC,CAACjC,EAAE,GAAG6B,KAAK,CAAC7B,EAAE;oCAChDC,QAAQF,MAAM+B,EAAE1C,KAAK,GAAGD,IAAIC,KAAK;oCACjCc,WAAWL,KAAKgB,GAAG,CAAC1B,IAAIG,IAAI;oCAClC,IAAIY,aAAaiC,aAAalC,QAAQC,UAAUL,KAAKiB,GAAG,CAAC3B,IAAIG,IAAI,EAAEW;gCACrE;;gCAPK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAQP;;wBAXK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAaL;;wBAAQ,qBAAGJ,KAAKqB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;2BAC1BE,KAAK,CAAC,GAAGG,KAAKW,CAAC,EACflD,GAAG,CAAC;qEAAEmD,eAAGpC;mCAAY;gCAAEX,MAAM+C;gCAAGd,YAAYC,IAAAA,mBAAQ,EAACvB;4BAAO;;;;;IACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, compareVectorScores, sampleEvenly } from '../vectors.ts';\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY \"path\", chunk');\n  const rows = (await stmt.all()) as Array<{\n    path: string;\n    chunk: number;\n    start_line: number;\n    end_line: number;\n    vector: Uint8Array;\n  }>;\n\n  let queryNorm = 0;\n  for (let d = 0; d < Math.min(storeDims, qv.length); d++) queryNorm += qv[d] * qv[d];\n  queryNorm = Math.sqrt(queryNorm);\n\n  const best = new Map<string, { score: number; chunk: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    // Rounding to int8 costs the stored row its unit norm, so dividing by that norm is what makes\n    // this a cosine rather than a scaled dot product. `scale` cancels out of the ratio.\n    let dot = 0;\n    let norm = 0;\n    const length = Math.min(q.length, qv.length);\n    for (let d = 0; d < length; d++) {\n      dot += q[d] * qv[d];\n      norm += q[d] * q[d];\n    }\n    for (let d = length; d < q.length; d++) norm += q[d] * q[d];\n    const score = norm === 0 || queryNorm === 0 ? 0 : dot / (Math.sqrt(norm) * queryNorm);\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score || (score === existing.score && row.chunk < existing.chunk)) {\n      best.set(row.path, { score, chunk: row.chunk, lines: `L${row.start_line}-${row.end_line}` });\n    }\n  }\n  return [...best.entries()]\n    .map(([path, value]) => ({ path, ...value }))\n    .sort(compareVectorScores)\n    .slice(0, fetch)\n    .map(({ path, lines, score }) => ({ path, lines, similarity: asCosine(score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  // Each side's norm divides its own dot product, so both are taken once here rather than per pair.\n  const target = sampleEvenly(targetRows).map((row) => {\n    const v = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    let norm = 0;\n    for (let d = 0; d < v.length; d++) norm += v[d] * v[d];\n    return { v, norm: Math.sqrt(norm) };\n  });\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    let otherNorm = 0;\n    for (let d = 0; d < other.length; d++) otherNorm += other[d] * other[d];\n    otherNorm = Math.sqrt(otherNorm);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = t.norm === 0 || otherNorm === 0 ? 0 : dot / (t.norm * otherNorm);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .map(([p, score]) => ({ path: p, score }))\n    .sort(compareVectorScores)\n    .slice(0, opts.k)\n    .map(({ path: p, score }) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","conn","rows","length","runBatch","map","row","scale","vector","path","chunk","qv","storeDims","fetch","allowed","stmt","queryNorm","d","best","q","dot","norm","score","existing","prepare","all","Math","min","sqrt","Map","has","Int8Array","buffer","byteOffset","byteLength","get","set","lines","start_line","end_line","entries","value","sort","compareVectorScores","slice","similarity","asCosine","opts","targetStmt","targetRows","target","other","otherNorm","t","len","sampleEvenly","v","exclude","undefined","p","k"],"mappings":";;;;;;;;;;;QAesBA;eAAAA;;QA2CAC;eAAAA;;QArDAC;eAAAA;;;yBAJsC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIrD,SAAeA,iBAAiBC,IAAgB,EAAEC,IAAsB;;;;;oBAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;oBACvB;;wBAAMF,KAAKG,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,SAACC;mCAAQ;gCAACA,IAAIC,KAAK;gCAAED,IAAIE,MAAM;gCAAEF,IAAIG,IAAI;gCAAEH,IAAII,KAAK;6BAAC;;;;oBAFhE;;;;;;IAIF;;AAIO,SAAeZ,eAAeG,IAAgB,EAAEU,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;;YACxHC,MACAb,MAQFc,WACKC,GAGHC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMZ,KAEHa,GAGFC,KACAC,MACElB,QACGc,IAIAA,IACHK,OACAC;;;;oBA5BK;;wBAAMtB,KAAKuB,OAAO,CAAC;;;oBAA1BT,OAAO;oBACC;;wBAAMA,KAAKU,GAAG;;;oBAAtBvB,OAAQ;oBAQVc,YAAY;oBAChB,IAASC,IAAI,GAAGA,IAAIS,KAAKC,GAAG,CAACf,WAAWD,GAAGR,MAAM,GAAGc,IAAKD,aAAaL,EAAE,CAACM,EAAE,GAAGN,EAAE,CAACM,EAAE;oBACnFD,YAAYU,KAAKE,IAAI,CAACZ;oBAEhBE,OAAO,IAAIW;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAa3B,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIQ,WAAW,CAACA,QAAQgB,GAAG,CAACxB,IAAIG,IAAI,GAAG;4BACjCU,IAAI,IAAIY,UAAUzB,IAAIE,MAAM,CAACwB,MAAM,EAAE1B,IAAIE,MAAM,CAACyB,UAAU,EAAEP,KAAKC,GAAG,CAACf,WAAWN,IAAIE,MAAM,CAAC0B,UAAU;4BAC3G,8FAA8F;4BAC9F,oFAAoF;4BAChFd,MAAM;4BACNC,OAAO;4BACLlB,SAASuB,KAAKC,GAAG,CAACR,EAAEhB,MAAM,EAAEQ,GAAGR,MAAM;4BAC3C,IAASc,KAAI,GAAGA,KAAId,QAAQc,KAAK;gCAC/BG,OAAOD,CAAC,CAACF,GAAE,GAAGN,EAAE,CAACM,GAAE;gCACnBI,QAAQF,CAAC,CAACF,GAAE,GAAGE,CAAC,CAACF,GAAE;4BACrB;4BACA,IAASA,KAAId,QAAQc,KAAIE,EAAEhB,MAAM,EAAEc,KAAKI,QAAQF,CAAC,CAACF,GAAE,GAAGE,CAAC,CAACF,GAAE;4BACrDK,QAAQD,SAAS,KAAKL,cAAc,IAAI,IAAII,MAAOM,CAAAA,KAAKE,IAAI,CAACP,QAAQL,SAAQ;4BAC7EO,WAAWL,KAAKiB,GAAG,CAAC7B,IAAIG,IAAI;4BAClC,IAAI,CAACc,YAAYD,QAAQC,SAASD,KAAK,IAAKA,UAAUC,SAASD,KAAK,IAAIhB,IAAII,KAAK,GAAGa,SAASb,KAAK,EAAG;gCACnGQ,KAAKkB,GAAG,CAAC9B,IAAIG,IAAI,EAAE;oCAAEa,OAAAA;oCAAOZ,OAAOJ,IAAII,KAAK;oCAAE2B,OAAO,AAAC,IAAqB/B,OAAlBA,IAAIgC,UAAU,EAAC,KAAgB,OAAbhC,IAAIiC,QAAQ;gCAAG;4BAC5F;wBACF;;wBAlBK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAmBL;;wBAAQ,qBAAGrB,KAAKsB,OAAO,IACpBnC,GAAG,CAAC;qEAAEI,kBAAMgC;mCAAY;gCAAEhC,MAAAA;+BAASgC;2BACnCC,IAAI,CAACC,8BAAmB,EACxBC,KAAK,CAAC,GAAG/B,OACTR,GAAG,CAAC;gCAAGI,aAAAA,MAAM4B,cAAAA,OAAOf,cAAAA;mCAAa;gCAAEb,MAAAA;gCAAM4B,OAAAA;gCAAOQ,YAAYC,IAAAA,mBAAQ,EAACxB;4BAAO;;;;;IACjF;;AAIO,SAAevB,YAAYE,IAAgB,EAAEQ,IAAY,EAAEsC,IAAgE;;YAC1HC,YACAC,YAGAC,QAOAnC,MACAb,MACAgB,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMZ,KAEH6C,OACFC,WACKnC,GAEJ,4BAAA,oBAAA,iBAAA,YAAA,QAAMoC,GACLjC,KACEkC,KACGrC,IACHK,OACAC;;;;oBAzBS;;wBAAMtB,KAAKuB,OAAO,CAAC;;;oBAAhCwB,aAAa;oBACC;;wBAAMA,WAAWvB,GAAG,CAAChB;;;oBAAnCwC,aAAc;oBACpB,IAAIA,WAAW9C,MAAM,KAAK,GAAG;;;;oBAC7B,kGAAkG;oBAC5F+C,SAASK,IAAAA,uBAAY,EAACN,YAAY5C,GAAG,CAAC,SAACC;wBAC3C,IAAMkD,IAAI,IAAIzB,UAAUzB,IAAIE,MAAM,CAACwB,MAAM,EAAE1B,IAAIE,MAAM,CAACyB,UAAU,EAAE3B,IAAIE,MAAM,CAAC0B,UAAU;wBACvF,IAAIb,OAAO;wBACX,IAAK,IAAIJ,IAAI,GAAGA,IAAIuC,EAAErD,MAAM,EAAEc,IAAKI,QAAQmC,CAAC,CAACvC,EAAE,GAAGuC,CAAC,CAACvC,EAAE;wBACtD,OAAO;4BAAEuC,GAAAA;4BAAGnC,MAAMK,KAAKE,IAAI,CAACP;wBAAM;oBACpC;oBAEa;;wBAAMpB,KAAKuB,OAAO,CAAC;;;oBAA1BT,OAAO;oBACC;;wBAAMA,KAAKU,GAAG;;;oBAAtBvB,OAAQ;oBACRgB,OAAO,IAAIW;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAa3B,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,MAAN;4BACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQsC,KAAKU,OAAO,CAAC3B,GAAG,CAACxB,IAAIG,IAAI,KAAMsC,KAAKjC,OAAO,IAAI,CAACiC,KAAKjC,OAAO,CAACgB,GAAG,CAACxB,IAAIG,IAAI,GAAI;4BAChG0C,QAAQ,IAAIpB,UAAUzB,IAAIE,MAAM,CAACwB,MAAM,EAAE1B,IAAIE,MAAM,CAACyB,UAAU,EAAE3B,IAAIE,MAAM,CAAC0B,UAAU;4BACvFkB,YAAY;4BAChB,IAASnC,IAAI,GAAGA,IAAIkC,MAAMhD,MAAM,EAAEc,IAAKmC,aAAaD,KAAK,CAAClC,EAAE,GAAGkC,KAAK,CAAClC,EAAE;4BACvEmC,YAAY1B,KAAKE,IAAI,CAACwB;4BACjB,mCAAA,4BAAA;;gCAAL,IAAK,aAAWF,6BAAX,8BAAA,SAAA,0BAAA,kCAAmB;oCAAbG,IAAN;oCACCjC,MAAM;oCACJkC,MAAM5B,KAAKC,GAAG,CAAC0B,EAAEG,CAAC,CAACrD,MAAM,EAAEgD,MAAMhD,MAAM;oCAC7C,IAASc,KAAI,GAAGA,KAAIqC,KAAKrC,KAAKG,OAAOiC,EAAEG,CAAC,CAACvC,GAAE,GAAGkC,KAAK,CAAClC,GAAE;oCAChDK,QAAQ+B,EAAEhC,IAAI,KAAK,KAAK+B,cAAc,IAAI,IAAIhC,MAAOiC,CAAAA,EAAEhC,IAAI,GAAG+B,SAAQ;oCACtE7B,WAAWL,KAAKiB,GAAG,CAAC7B,IAAIG,IAAI;oCAClC,IAAIc,aAAamC,aAAapC,QAAQC,UAAUL,KAAKkB,GAAG,CAAC9B,IAAIG,IAAI,EAAEa;gCACrE;;gCAPK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAQP;;wBAdK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAgBL;;wBAAQ,qBAAGJ,KAAKsB,OAAO,IACpBnC,GAAG,CAAC;qEAAEsD,eAAGrC;mCAAY;gCAAEb,MAAMkD;gCAAGrC,OAAAA;4BAAM;2BACtCoB,IAAI,CAACC,8BAAmB,EACxBC,KAAK,CAAC,GAAGG,KAAKa,CAAC,EACfvD,GAAG,CAAC;gCAAGI,AAAMkD,UAANlD,MAASa,cAAAA;mCAAa;gCAAEb,MAAMkD;gCAAGd,YAAYC,IAAAA,mBAAQ,EAACxB;4BAAO;;;;;IACzE"}

package/dist/cjs/store/turso/connection.js

@@ -30,6 +30,7 @@ var _batchts = require("../batch.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _nativets = require("./native.js");
+var _sqlfunctionsts = require("./sql-functions.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -729,7 +730,7 @@ function createConnection(db) {
                             _ = TursoStatementWrapper.bind;
                             return [
                                 4,
-                                db.prepare(sql)
+                                db.prepare((0, _sqlfunctionsts.rewriteFunctions)(sql))
                             ];
                         case 1:
                             return [

package/dist/cjs/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["cacheFilePath","checkpointWal","createConnection","fileSize","reclaimSpace","TursoStatementWrapper","stmt","run","params","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","db","err","exec","console","error","message","conn","rows","prepare","find","r","name","file","path","stat","size","turso","tursoApi","connect","CONNECT_OPTS","experimental","setMeta","String","close","sql","runBatch","paramRows","length","withTransaction","rewritten","row","rewriteInsert","flat","BEGIN_WRITE"],"mappings":";;;;;;;;;;;QA0DsBA;eAAAA;;QAVAC;eAAAA;;QA2CNC;eAAAA;;QA5BMC;eAAAA;;QAUAC;eAAAA;;;wBAzED;uBAES;wBACN;6BACqB;wBAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAMvC,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,sCAAN;;aAAMA,sBAGQC,IAAoB;gCAH5BD;QAIF,IAAI,CAACC,IAAI,GAAGA;;iBAJVD;IAOJ,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACC,GAAG,OAAb,YAAc,qBAAGC;;;QAC1B;;IAEA,OAAMC,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGD,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACG,GAAG,OAAb,YAAc,qBAAGD;;;QAC1B;;IAEA,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGF,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACI,GAAG,OAAb,YAAc,qBAAGF;;;QAC1B;;IAEA,OAAOG,OAEN,GAFD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGH,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAC1B;;;;wBAAP;;uCAAA,0CAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACK,OAAO,OAAjB,YAAkB,qBAAGH;;;wBAA5B;;;;;;QACF;;IAEAI,OAAAA,OAEC,GAFDA,SAAAA;QACE,OAAO,IAAI,CAACN,IAAI,CAACM,OAAO;IAC1B;IAEAC,OAAAA,cAEC,GAFDA,SAAAA,eAAeC,OAAgB;QAC7B,IAAI,CAACR,IAAI,CAACS,YAAY,CAACD;IACzB;WA7BIT;;AAkCC,SAAeJ,cAAce,EAAY;;YAGrCC;;;;;;;;;;oBADP;;wBAAMD,GAAGE,IAAI,CAAC;;;oBAAd;;;;;;oBACOD;oBACPE,QAAQC,KAAK,CAAC,AAAC,2FAAiH,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEnI;;AAIO,SAAerB,cAAcsB,IAAgB;;YAE3CC,YADDA;;;;oBAAe;;wBAAMD,KAAKE,OAAO,CAAC;;;oBAA1B;;wBAAO,cAA4Cd,GAAG;;;oBAA9Da,OAAQ;oBACd;;wBAAOA,EAAAA,aAAAA,KAAKE,IAAI,CAAC,SAACC;mCAAMA,EAAEC,IAAI,KAAK;wCAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;;;;IACtD;;AAEO,SAAezB,SAAS0B,IAAY;;;;;;;;;;;;oBAE/B;;wBAAMC,IAAAA,cAAI,EAACD;;;oBAAnB;;wBAAQ,cAAkBE,IAAI;;;;oBAE9B;;wBAAO;;;;;;;;IAEX;;AAIO,SAAe3B,aAAayB,IAAY;;YAErCG,OACAhB,IAGEe,MAODd;;;;;;;;;;oBAXO;;wBAAMgB,IAAAA,kBAAQ;;;oBAAtBD,QAAQ;oBACH;;wBAAMA,MAAME,OAAO,CAACL,MAAM,wCAAKM,sBAAY;4BAAEC,cAAc,AAAC,qBAAGD,sBAAY,CAACC,YAAY;gCAAE;;;;;oBAA/FpB,KAAK;;;;;;;;;oBAET;;wBAAMA,GAAGE,IAAI,CAAC;;;oBAAd;oBACa;;wBAAMf,SAAS0B;;;oBAAtBE,OAAO;yBAETA,CAAAA,SAAS,IAAG,GAAZA;;;;oBAAe;;wBAAMM,IAAAA,iBAAO,EAACnC,iBAAiBc,KAAK,gBAAgBsB,OAAOP;;;oBAA3D;;;oBACnB;;wBAAM9B,cAAce;;;oBAApB;;;;;;oBAEA;;wBAAMA,GAAGuB,KAAK;;;oBAAd;;;;;;;;;;oBAEKtB;oBACPE,QAAQC,KAAK,CAAC,AAAC,iFAAuG,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEzH;;AAEO,SAASnB,iBAAiBc,EAAY;IAC3C,IAAMM,OAAmB;QACjBJ,MAAN,SAAMA,KAAKsB,GAAW;;;;;4BACpB;;gCAAMxB,GAAGE,IAAI,CAACsB;;;4BAAd;;;;;;YACF;;QACMhB,SAAN,SAAMA,QAAQgB,GAAW;;;;;;gCACZnC;4BAAsB;;gCAAMW,GAAGQ,OAAO,CAACgB;;;4BAAlD;;gCAAO,IAAA,CAAA,EAAA,MAAInC;;oCAAsB;kCAAqB;;;;YACxD;;QAGMoC,UAFN,gGAAgG;QAChG,wGAAwG;QACxG,SAAMA,SAASD,GAAW,EAAEE,SAAsB;;;;;4BAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMC,IAAAA,8BAAe,EACnBtB,MACA;;4CACQuB,WAEEvC,MAYFA,OAE+BA,OAA9B,2BAAA,mBAAA,gBAAA,WAAA,OAAMwC;;;;oDAhBPD,YAAYE,IAAAA,sBAAa,EAACP,KAAKE,UAAUC,MAAM;yDACjDE,WAAAA;;;;oDACW;;wDAAM7B,GAAGQ,OAAO,CAACqB,UAAUL,GAAG;;;oDAArClC,OAAO;;;;;;;;;oDAEX,uFAAuF;oDACvF,uFAAuF;oDACvF;;wDAAMA,KAAKC,GAAG,CAACmC,UAAUM,IAAI;;;oDAA7B;;;;;;oDAEA;;wDAAM1C,KAAKiC,KAAK;;;oDAAhB;;;;;oDAEF;;;;oDAIW;;wDAAMvB,GAAGQ,OAAO,CAACgB;;;oDAAxBlC,QAAO;;;;;;;;;oDAEN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaoC;;;2DAAb,6BAAA,QAAA;;;;oDAAMI,MAAN;oDAAwB;;wDAAMxC,CAAAA,QAAAA,OAAKC,GAAG,OAARD,OAAS,qBAAGwC;;;oDAAlB;;;oDAAxB;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAEL;;wDAAMxC,MAAKiC,KAAK;;;oDAAhB;;;;;;;;;;oCAEJ;mCACAU,0BAAW;;;4BAxBb;;;;;;YA0BF;;IACF;IACA,OAAO3B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and nothing else owns this statement's lifetime.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["cacheFilePath","checkpointWal","createConnection","fileSize","reclaimSpace","TursoStatementWrapper","stmt","run","params","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","db","err","exec","console","error","message","conn","rows","prepare","find","r","name","file","path","stat","size","turso","tursoApi","connect","CONNECT_OPTS","experimental","setMeta","String","close","sql","rewriteFunctions","runBatch","paramRows","length","withTransaction","rewritten","row","rewriteInsert","flat","BEGIN_WRITE"],"mappings":";;;;;;;;;;;QA2DsBA;eAAAA;;QAVAC;eAAAA;;QA2CNC;eAAAA;;QA5BMC;eAAAA;;QAUAC;eAAAA;;;wBA1ED;uBAES;wBACN;6BACqB;wBAEN;8BACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAMjC,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,sCAAN;;aAAMA,sBAGQC,IAAoB;gCAH5BD;QAIF,IAAI,CAACC,IAAI,GAAGA;;iBAJVD;IAOJ,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACC,GAAG,OAAb,YAAc,qBAAGC;;;QAC1B;;IAEA,OAAMC,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGD,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACG,GAAG,OAAb,YAAc,qBAAGD;;;QAC1B;;IAEA,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGF,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACI,GAAG,OAAb,YAAc,qBAAGF;;;QAC1B;;IAEA,OAAOG,OAEN,GAFD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGH,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAC1B;;;;wBAAP;;uCAAA,0CAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACK,OAAO,OAAjB,YAAkB,qBAAGH;;;wBAA5B;;;;;;QACF;;IAEAI,OAAAA,OAEC,GAFDA,SAAAA;QACE,OAAO,IAAI,CAACN,IAAI,CAACM,OAAO;IAC1B;IAEAC,OAAAA,cAEC,GAFDA,SAAAA,eAAeC,OAAgB;QAC7B,IAAI,CAACR,IAAI,CAACS,YAAY,CAACD;IACzB;WA7BIT;;AAkCC,SAAeJ,cAAce,EAAY;;YAGrCC;;;;;;;;;;oBADP;;wBAAMD,GAAGE,IAAI,CAAC;;;oBAAd;;;;;;oBACOD;oBACPE,QAAQC,KAAK,CAAC,AAAC,2FAAiH,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEnI;;AAIO,SAAerB,cAAcsB,IAAgB;;YAE3CC,YADDA;;;;oBAAe;;wBAAMD,KAAKE,OAAO,CAAC;;;oBAA1B;;wBAAO,cAA4Cd,GAAG;;;oBAA9Da,OAAQ;oBACd;;wBAAOA,EAAAA,aAAAA,KAAKE,IAAI,CAAC,SAACC;mCAAMA,EAAEC,IAAI,KAAK;wCAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;;;;IACtD;;AAEO,SAAezB,SAAS0B,IAAY;;;;;;;;;;;;oBAE/B;;wBAAMC,IAAAA,cAAI,EAACD;;;oBAAnB;;wBAAQ,cAAkBE,IAAI;;;;oBAE9B;;wBAAO;;;;;;;;IAEX;;AAIO,SAAe3B,aAAayB,IAAY;;YAErCG,OACAhB,IAGEe,MAODd;;;;;;;;;;oBAXO;;wBAAMgB,IAAAA,kBAAQ;;;oBAAtBD,QAAQ;oBACH;;wBAAMA,MAAME,OAAO,CAACL,MAAM,wCAAKM,sBAAY;4BAAEC,cAAc,AAAC,qBAAGD,sBAAY,CAACC,YAAY;gCAAE;;;;;oBAA/FpB,KAAK;;;;;;;;;oBAET;;wBAAMA,GAAGE,IAAI,CAAC;;;oBAAd;oBACa;;wBAAMf,SAAS0B;;;oBAAtBE,OAAO;yBAETA,CAAAA,SAAS,IAAG,GAAZA;;;;oBAAe;;wBAAMM,IAAAA,iBAAO,EAACnC,iBAAiBc,KAAK,gBAAgBsB,OAAOP;;;oBAA3D;;;oBACnB;;wBAAM9B,cAAce;;;oBAApB;;;;;;oBAEA;;wBAAMA,GAAGuB,KAAK;;;oBAAd;;;;;;;;;;oBAEKtB;oBACPE,QAAQC,KAAK,CAAC,AAAC,iFAAuG,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEzH;;AAEO,SAASnB,iBAAiBc,EAAY;IAC3C,IAAMM,OAAmB;QACjBJ,MAAN,SAAMA,KAAKsB,GAAW;;;;;4BACpB;;gCAAMxB,GAAGE,IAAI,CAACsB;;;4BAAd;;;;;;YACF;;QACMhB,SAAN,SAAMA,QAAQgB,GAAW;;;;;;gCACZnC;4BAAsB;;gCAAMW,GAAGQ,OAAO,CAACiB,IAAAA,gCAAgB,EAACD;;;4BAAnE;;gCAAO,IAAA,CAAA,EAAA,MAAInC;;oCAAsB;kCAAuC;;;;YAC1E;;QAGMqC,UAFN,gGAAgG;QAChG,wGAAwG;QACxG,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMC,IAAAA,8BAAe,EACnBvB,MACA;;4CACQwB,WAEExC,MAYFA,OAE+BA,OAA9B,2BAAA,mBAAA,gBAAA,WAAA,OAAMyC;;;;oDAhBPD,YAAYE,IAAAA,sBAAa,EAACR,KAAKG,UAAUC,MAAM;yDACjDE,WAAAA;;;;oDACW;;wDAAM9B,GAAGQ,OAAO,CAACsB,UAAUN,GAAG;;;oDAArClC,OAAO;;;;;;;;;oDAEX,uFAAuF;oDACvF,uFAAuF;oDACvF;;wDAAMA,KAAKC,GAAG,CAACoC,UAAUM,IAAI;;;oDAA7B;;;;;;oDAEA;;wDAAM3C,KAAKiC,KAAK;;;oDAAhB;;;;;oDAEF;;;;oDAIW;;wDAAMvB,GAAGQ,OAAO,CAACgB;;;oDAAxBlC,QAAO;;;;;;;;;oDAEN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaqC;;;2DAAb,6BAAA,QAAA;;;;oDAAMI,MAAN;oDAAwB;;wDAAMzC,CAAAA,QAAAA,OAAKC,GAAG,OAARD,OAAS,qBAAGyC;;;oDAAlB;;;oDAAxB;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAEL;;wDAAMzC,MAAKiC,KAAK;;;oDAAhB;;;;;;;;;;oCAEJ;mCACAW,0BAAW;;;4BAxBb;;;;;;YA0BF;;IACF;IACA,OAAO5B;AACT"}

package/dist/cjs/store/turso/lexical-text.d.cts

@@ -0,0 +1 @@
+export declare function stemFolded(text: string): string;

package/dist/cjs/store/turso/lexical-text.d.ts

@@ -0,0 +1 @@
+export declare function stemFolded(text: string): string;

package/dist/cjs/store/turso/lexical-text.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "stemFolded", {
+    enumerable: true,
+    get: function() {
+        return stemFolded;
+    }
+});
+var _stemmer = require("stemmer");
+var _segmentts = require("../../text/segment.js");
+function stemFolded(text) {
+    var out = '';
+    var end = 0;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = (0, _segmentts.searchTokens)(text)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var token = _step.value;
+            out += text.slice(end, token.start).replace(RegExp("[\\u0027\\u2019]", "gu"), ' ');
+            out += token.unspaced ? token.text : (0, _stemmer.stemmer)((0, _segmentts.foldForSearch)(token.text));
+            end = token.end;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return out + text.slice(end).replace(RegExp("[\\u0027\\u2019]", "gu"), ' ');
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/turso/lexical-text.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical-text.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\nimport { foldForSearch, searchTokens } from '../../text/segment.ts';\n\nexport function stemFolded(text: string): string {\n  let out = '';\n  let end = 0;\n  for (const token of searchTokens(text)) {\n    out += text.slice(end, token.start).replace(/[\\u0027\\u2019]/gu, ' ');\n    out += token.unspaced ? token.text : stemmer(foldForSearch(token.text));\n    end = token.end;\n  }\n  return out + text.slice(end).replace(/[\\u0027\\u2019]/gu, ' ');\n}\n"],"names":["stemFolded","text","out","end","searchTokens","token","slice","start","replace","unspaced","stemmer","foldForSearch"],"mappings":";;;;+BAGgBA;;;eAAAA;;;uBAHQ;yBACoB;AAErC,SAASA,WAAWC,IAAY;IACrC,IAAIC,MAAM;IACV,IAAIC,MAAM;QACL,kCAAA,2BAAA;;QAAL,QAAK,YAAeC,IAAAA,uBAAY,EAACH,0BAA5B,SAAA,6BAAA,QAAA,yBAAA,iCAAmC;YAAnC,IAAMI,QAAN;YACHH,OAAOD,KAAKK,KAAK,CAACH,KAAKE,MAAME,KAAK,EAAEC,OAAO,CAAC,kCAAoB;YAChEN,OAAOG,MAAMI,QAAQ,GAAGJ,MAAMJ,IAAI,GAAGS,IAAAA,gBAAO,EAACC,IAAAA,wBAAa,EAACN,MAAMJ,IAAI;YACrEE,MAAME,MAAMF,GAAG;QACjB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD,MAAMD,KAAKK,KAAK,CAACH,KAAKK,OAAO,CAAC,kCAAoB;AAC3D"}

package/dist/cjs/store/turso/lexical.js

@@ -12,6 +12,7 @@ Object.defineProperty(exports, "queryLexical", {
 });
 var _errorsts = require("../../errors.js");
 var _segmentts = require("../../text/segment.js");
+var _lexicaltextts = require("./lexical-text.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -165,15 +166,20 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
 var FIELDS = [
-    'title',
-    'summary',
-    'text'
+    'title_stem',
+    'summary_stem',
+    'text_stem'
 ];
 var NGRAM_FIELDS = [
     'title_ngram',
     'summary_ngram',
     'text_ngram'
 ];
+var AUTHORED_FIELDS = [
+    'title',
+    'summary',
+    'text'
+];
 // FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:
 // no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.
 var FTS5_OPERATORS = [
@@ -232,11 +238,21 @@ function splitTerms(terms) {
     var words = [];
     var phrases = [];
     var unspaced = [];
+    var emptyPhrase = false;
     var withoutPhrases = terms.replace(/"([^"]*)"/g, function(_m, inner) {
+        var _unspaced;
         var phrase = inner.trim();
-        if (phrase.length === 0) return ' ';
-        if ((0, _segmentts.hasUnspacedRun)(phrase)) unspaced.push(phrase);
-        else phrases.push(phrase);
+        var tokens = (0, _segmentts.searchTokens)(phrase);
+        if (tokens.length === 0) {
+            emptyPhrase = true;
+            return ' ';
+        }
+        (_unspaced = unspaced).push.apply(_unspaced, _to_consumable_array(tokens.filter(function(token) {
+            return token.unspaced;
+        }).map(function(token) {
+            return token.text;
+        })));
+        phrases.push(tokens);
         return ' ';
     });
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
@@ -244,8 +260,27 @@ function splitTerms(terms) {
         for(var _iterator = withoutPhrases.split(/\s+/)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var tok = _step.value;
             if (tok.length === 0) continue;
-            if ((0, _segmentts.hasUnspacedRun)(tok)) unspaced.push(tok);
-            else words.push(tok);
+            var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+            try {
+                for(var _iterator1 = (0, _segmentts.searchTokens)(tok)[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                    var token = _step1.value;
+                    if (token.unspaced) unspaced.push(token.text);
+                    else words.push(token.text);
+                }
+            } catch (err) {
+                _didIteratorError1 = true;
+                _iteratorError1 = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                        _iterator1.return();
+                    }
+                } finally{
+                    if (_didIteratorError1) {
+                        throw _iteratorError1;
+                    }
+                }
+            }
         }
     } catch (err) {
         _didIteratorError = true;
@@ -264,7 +299,8 @@ function splitTerms(terms) {
     return {
         words: words,
         phrases: phrases,
-        unspaced: unspaced
+        unspaced: unspaced,
+        emptyPhrase: emptyPhrase
     };
 }
 // fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays
@@ -275,7 +311,20 @@ function escapeFtsLiteral(query) {
 // Terms are quoted ("O'Brien's" is a parse error bare) and AND-joined: Tantivy's default for
 // bare terms is disjunctive, so space-joining would loosen "apple banana" into "apple OR banana".
 function buildQueries(words, phrases, unspaced) {
-    var defaultParts = _to_consumable_array(words).concat(_to_consumable_array(phrases)).map(function(t) {
+    var phraseParts = phrases.flatMap(function(phrase) {
+        var latin = phrase.filter(function(token) {
+            return !token.unspaced;
+        }).map(function(token) {
+            return (0, _lexicaltextts.stemFolded)(token.text);
+        });
+        if (latin.length === 0) return [];
+        return phrase.some(function(token) {
+            return token.unspaced;
+        }) ? latin : [
+            latin.join(' ')
+        ];
+    });
+    var defaultParts = _to_consumable_array(words.map(_lexicaltextts.stemFolded)).concat(_to_consumable_array(phraseParts)).map(function(t) {
         return '"'.concat(t, '"');
     });
     var unspacedParts = unspaced.map(function(t) {
@@ -297,7 +346,7 @@ function ftsBranch(cols, query, opts) {
 }
 function queryLexical(conn, terms, opts) {
     return _async_to_generator(function() {
-        var unsupported, _splitTerms, words, phrases, unspaced, _buildQueries, defaultQuery, ngramQuery, branches, sql, stmt, rows;
+        var _stmt, unsupported, _splitTerms, words, phrases, unspaced, emptyPhrase, _buildQueries, defaultQuery, ngramQuery, branches, ranked, phraseSelect, phraseJoin, sql, stmt, rows, matches;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -305,7 +354,11 @@ function queryLexical(conn, terms, opts) {
                     if (unsupported !== null) {
                         throw new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "turso" does not implement FTS5\'s '.concat(unsupported.label, ' ("').concat(unsupported.token, '") in this build; rephrase "').concat(terms.trim(), '" without it, or set "store" to "sqlite" in this tree\'s config to search it as written'));
                     }
-                    _splitTerms = splitTerms(terms), words = _splitTerms.words, phrases = _splitTerms.phrases, unspaced = _splitTerms.unspaced;
+                    _splitTerms = splitTerms(terms), words = _splitTerms.words, phrases = _splitTerms.phrases, unspaced = _splitTerms.unspaced, emptyPhrase = _splitTerms.emptyPhrase;
+                    if (emptyPhrase) return [
+                        2,
+                        []
+                    ];
                     _buildQueries = buildQueries(words, phrases, unspaced), defaultQuery = _buildQueries.defaultQuery, ngramQuery = _buildQueries.ngramQuery;
                     if (defaultQuery === '' && ngramQuery === '') return [
                         2,
@@ -315,8 +368,13 @@ function queryLexical(conn, terms, opts) {
                     if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));
                     if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));
                     // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,
-                    // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.
-                    sql = "SELECT path, NULL AS hit FROM (\n    SELECT path, SUM(score) AS score FROM (\n      ".concat(branches.join(' UNION ALL '), "\n    ) parts GROUP BY path HAVING COUNT(*) = ").concat(branches.length, "\n  ) sq ORDER BY score DESC, path LIMIT ?");
+                    // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase
+                    // candidates carry authored fields through the join below so the shared verifier can reject
+                    // cross-field, reordered, and non-adjacent matches before applying the caller's limit.
+                    ranked = "SELECT path, SUM(score) AS score FROM (\n      ".concat(branches.join(' UNION ALL '), "\n    ) parts GROUP BY path HAVING COUNT(*) = ").concat(branches.length);
+                    phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
+                    phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';
+                    sql = "SELECT sq.path, sq.score".concat(phraseSelect, " FROM (\n    ").concat(ranked, "\n  ) sq").concat(phraseJoin, " ORDER BY sq.score DESC, sq.path").concat(phrases.length > 0 ? '' : ' LIMIT ?');
                     return [
                         4,
                         conn.prepare(sql)
@@ -325,7 +383,9 @@ function queryLexical(conn, terms, opts) {
                     stmt = _state.sent();
                     return [
                         4,
-                        stmt.all(opts.limit)
+                        (_stmt = stmt).all.apply(_stmt, _to_consumable_array(phrases.length > 0 ? [] : [
+                            opts.limit
+                        ]))
                     ];
                 case 2:
                     rows = _state.sent();
@@ -336,9 +396,22 @@ function queryLexical(conn, terms, opts) {
                     })) {
                         throw new _errorsts.SenseError('LEXICAL_NULL_PATH', 'store "turso" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');
                     }
+                    matches = phrases.length === 0 ? rows : rows.filter(function(row) {
+                        return phrases.every(function(phrase) {
+                            return AUTHORED_FIELDS.some(function(field) {
+                                var _row_field;
+                                return (0, _segmentts.matchesSearchPhrase)((_row_field = row[field]) !== null && _row_field !== void 0 ? _row_field : '', phrase);
+                            });
+                        });
+                    });
                     return [
                         2,
-                        rows
+                        matches.slice(0, opts.limit).map(function(param) {
+                            var path = param.path;
+                            return {
+                                path: path
+                            };
+                        })
                     ];
             }
         });

package/dist/cjs/store/turso/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: string[]; unspaced: string[] } {\n  const words: string[] = [];\n  const phrases: string[] = [];\n  const unspaced: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    if (hasUnspacedRun(phrase)) unspaced.push(phrase);\n    else phrases.push(phrase);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) unspaced.push(tok);\n    else words.push(tok);\n  }\n  return { words, phrases, unspaced };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: string[], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const defaultParts = [...words, ...phrases].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `hit` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced } = splitTerms(terms);\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(opts.limit)) as unknown as LexicalHit[];\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  return rows;\n}\n"],"names":["queryLexical","FIELDS","NGRAM_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","_m","inner","phrase","length","hasUnspacedRun","push","split","tok","escapeFtsLiteral","query","buildQueries","defaultParts","map","t","unspacedParts","defaultQuery","join","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","conn","unsupported","branches","sql","stmt","rows","SenseError","prepare","all","limit","some","r","path"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;;;;;+BAyEjFA;;;eAAAA;;;wBAxEK;yBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAC3C,IAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AAEnE,uFAAuF;AACvF,+FAA+F;AAC/F,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,UAAoB,EAAE;IAC5B,IAAMC,WAAqB,EAAE;IAC7B,IAAMT,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACS,IAAIC;QACtD,IAAMC,SAASD,MAAMN,IAAI;QACzB,IAAIO,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChC,IAAIC,IAAAA,yBAAc,EAACF,SAASH,SAASM,IAAI,CAACH;aACrCJ,QAAQO,IAAI,CAACH;QAClB,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaZ,eAAegB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;YACtB,IAAIC,IAAAA,yBAAc,EAACG,MAAMR,SAASM,IAAI,CAACE;iBAClCV,MAAMQ,IAAI,CAACE;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,SAAAA;QAASC,UAAAA;IAAS;AACpC;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASS,iBAAiBC,KAAa;IACrC,OAAOA,MAAMlB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASmB,aAAab,KAAe,EAAEC,OAAiB,EAAEC,QAAkB;IAC1E,IAAMY,eAAe,AAAC,qBAAGd,cAAO,qBAAGC,UAASc,GAAG,CAAC,SAACC;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAC7D,IAAMC,gBAAgBf,SAASa,GAAG,CAAC,SAACC;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAChD,OAAO;QAAEE,cAAcJ,aAAaK,IAAI,CAAC;QAAUC,YAAYH,cAAcE,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASE,UAAUC,IAAuB,EAAEV,KAAa,EAAEW,IAAyB;IAClF,IAAMC,UAAUF,KAAKP,GAAG,CAAC,SAACU;eAAM,AAAC,WAAY,OAAFA;OAAKN,IAAI,CAAC;IACrD,IAAMO,MAAMf,iBAAiBC;IAC7B,OAAO,AAAC,4CAAwDc,OAAbF,SAAQ,OAC1CD,OAD+CG,KAAI,kCACjBF,OAAlCD,KAAKI,SAAS,EAAC,qBAAgCD,OAAbF,SAAQ,OAAcD,OAATG,KAAI,OAAuBH,OAAlBA,KAAKK,SAAS,EAAC,KAAkB,OAAfL,KAAKM,SAAS;AAC3G;AAIO,SAAe5C,aAAa6C,IAAgB,EAAEtC,KAAa,EAAE+B,IAAyB;;YACrFQ,aAI+BhC,aAA7BC,OAAOC,SAASC,UACaW,eAA7BK,cAAcE,YAGhBY,UAMAC,KAKAC,MACAC;;;;oBApBAJ,cAAcxC,oBAAoBC;oBACxC,IAAIuC,gBAAgB,MAAM;wBACxB,MAAM,IAAIK,oBAAU,CAAC,4BAA4B,AAAC,4CAAiEL,OAAvBA,YAAY1C,KAAK,EAAC,OAAqDG,OAAhDuC,YAAYlC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBAClL;oBACqCC,cAAAA,WAAWP,QAAxCQ,QAA6BD,YAA7BC,OAAOC,UAAsBF,YAAtBE,SAASC,WAAaH,YAAbG;oBACaW,gBAAAA,aAAab,OAAOC,SAASC,WAA1DgB,eAA6BL,cAA7BK,cAAcE,aAAeP,cAAfO;oBACtB,IAAIF,iBAAiB,MAAME,eAAe,IAAI;;;;oBAExCY;oBACN,IAAId,iBAAiB,IAAIc,SAASxB,IAAI,CAACa,UAAUnC,QAAQgC,cAAcK;oBACvE,IAAIH,eAAe,IAAIY,SAASxB,IAAI,CAACa,UAAUlC,cAAciC,YAAYG;oBAEzE,8FAA8F;oBAC9F,mFAAmF;oBAC7EU,MAAM,AAAC,uFAG+BD,OADtCA,SAASb,IAAI,CAAC,gBAAe,kDACyB,OAAhBa,SAAS1B,MAAM,EAAC;oBAE/C;;wBAAMwB,KAAKO,OAAO,CAACJ;;;oBAA1BC,OAAO;oBACC;;wBAAMA,KAAKI,GAAG,CAACf,KAAKgB,KAAK;;;oBAAjCJ,OAAQ;oBACd,+FAA+F;oBAC/F,yFAAyF;oBACzF,IAAIA,KAAKK,IAAI,CAAC,SAACC;+BAAMA,EAAEC,IAAI,KAAK;wBAAO;wBACrC,MAAM,IAAIN,oBAAU,CAAC,qBAAqB;oBAC5C;oBACA;;wBAAOD;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const phrases: SearchToken[][] = [];\n  const unspaced: string[] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const tokens = searchTokens(phrase);\n    if (tokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n    phrases.push(tokens);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) unspaced.push(token.text);\n      else words.push(token.text);\n    }\n  }\n  return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const phraseParts = phrases.flatMap((phrase) => {\n    const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n    if (latin.length === 0) return [];\n    return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n  });\n  const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase) return [];\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n  // candidates carry authored fields through the join below so the shared verifier can reject\n  // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n  const ranked = `SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n  const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n  const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n    ${ranked}\n  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["queryLexical","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","searchTokens","length","push","filter","map","text","split","tok","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","stemFolded","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","conn","stmt","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","rows","matches","SenseError","prepare","all","limit","r","path","row","every","field","matchesSearchPhrase","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;;;;;+BAwFjFA;;;eAAAA;;;wBAvFK;yBACyC;6BAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE3B,IAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,IAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,IAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,UAA2B,EAAE;IACnC,IAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,IAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACU,IAAIC;YAOtDH;QANA,IAAMI,SAASD,MAAMP,IAAI;QACzB,IAAMS,SAASC,IAAAA,uBAAY,EAACF;QAC5B,IAAIC,OAAOE,MAAM,KAAK,GAAG;YACvBN,cAAc;YACd,OAAO;QACT;QACAD,CAAAA,YAAAA,UAASQ,IAAI,OAAbR,WAAc,qBAAGK,OAAOI,MAAM,CAAC,SAACd;mBAAUA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUA,MAAMgB,IAAI;;QACnFZ,QAAQS,IAAI,CAACH;QACb,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAad,eAAeqB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIN,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACO,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAMlB,QAAN;oBACH,IAAIA,MAAMK,QAAQ,EAAEA,SAASQ,IAAI,CAACb,MAAMgB,IAAI;yBACvCb,MAAMU,IAAI,CAACb,MAAMgB,IAAI;gBAC5B;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAIP;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,OAAO;QAAEb,OAAAA;QAAOC,SAAAA;QAASC,UAAAA;QAAUC,aAAAA;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASa,iBAAiBC,KAAa;IACrC,OAAOA,MAAMvB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASwB,aAAalB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,IAAMiB,cAAclB,QAAQmB,OAAO,CAAC,SAACd;QACnC,IAAMe,QAAQf,OAAOK,MAAM,CAAC,SAACd;mBAAU,CAACA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUyB,IAAAA,yBAAU,EAACzB,MAAMgB,IAAI;;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOH,OAAOiB,IAAI,CAAC,SAAC1B;mBAAUA,MAAMK,QAAQ;aAAImB,QAAQ;YAACA,MAAMG,IAAI,CAAC;SAAK;IAC3E;IACA,IAAMC,eAAe,AAAC,qBAAGzB,MAAMY,GAAG,CAACU,yBAAU,UAAG,qBAAGH,cAAaP,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IACjF,IAAMC,gBAAgBzB,SAASU,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAChD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEd,KAAa,EAAEe,IAAyB;IAClF,IAAMC,UAAUF,KAAKnB,GAAG,CAAC,SAACsB;eAAM,AAAC,WAAY,OAAFA;OAAKV,IAAI,CAAC;IACrD,IAAMW,MAAMnB,iBAAiBC;IAC7B,OAAO,AAAC,4CAAwDkB,OAAbF,SAAQ,OAC1CD,OAD+CG,KAAI,kCACjBF,OAAlCD,KAAKI,SAAS,EAAC,qBAAgCD,OAAbF,SAAQ,OAAcD,OAATG,KAAI,OAAuBH,OAAlBA,KAAKK,SAAS,EAAC,KAAkB,OAAfL,KAAKM,SAAS;AAC3G;AAIO,SAAetD,aAAauD,IAAgB,EAAE/C,KAAa,EAAEwC,IAAyB;;YA2BvEQ,OA1BdC,aAI4C1C,aAA1CC,OAAOC,SAASC,UAAUC,aAEGe,eAA7BU,cAAcC,YAGhBa,UAQAC,QAGAC,cACAC,YACAC,KAGAN,MACAO,MAMAC;;;;oBAhCAP,cAAclD,oBAAoBC;oBACxC,IAAIiD,gBAAgB,MAAM;wBACxB,MAAM,IAAIQ,oBAAU,CAAC,4BAA4B,AAAC,4CAAiER,OAAvBA,YAAYpD,KAAK,EAAC,OAAqDG,OAAhDiD,YAAY5C,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBAClL;oBACkDC,cAAAA,WAAWP,QAArDQ,QAA0CD,YAA1CC,OAAOC,UAAmCF,YAAnCE,SAASC,WAA0BH,YAA1BG,UAAUC,cAAgBJ,YAAhBI;oBAClC,IAAIA,aAAa;;;;oBACoBe,gBAAAA,aAAalB,OAAOC,SAASC,WAA1D0B,eAA6BV,cAA7BU,cAAcC,aAAeX,cAAfW;oBACtB,IAAID,iBAAiB,MAAMC,eAAe,IAAI;;;;oBAExCa;oBACN,IAAId,iBAAiB,IAAIc,SAAShC,IAAI,CAACoB,UAAU7C,QAAQ2C,cAAcI;oBACvE,IAAIH,eAAe,IAAIa,SAAShC,IAAI,CAACoB,UAAU5C,cAAc2C,YAAYG;oBAEzE,8FAA8F;oBAC9F,0FAA0F;oBAC1F,4FAA4F;oBAC5F,uFAAuF;oBACjFW,SAAS,AAAC,kDAE4BD,OADtCA,SAASlB,IAAI,CAAC,gBAAe,kDACyB,OAAhBkB,SAASjC,MAAM;oBACrDmC,eAAe3C,QAAQQ,MAAM,GAAG,IAAI,mDAAmD;oBACvFoC,aAAa5C,QAAQQ,MAAM,GAAG,IAAI,4CAA4C;oBAC9EqC,MAAM,AAAC,2BACTH,OADmCC,cAAa,iBAE9CC,OADFF,QAAO,YACwC1C,OAA7C4C,YAAW,oCAAuE,OAArC5C,QAAQQ,MAAM,GAAG,IAAI,KAAK;oBAChE;;wBAAM8B,KAAKW,OAAO,CAACJ;;;oBAA1BN,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKW,GAAG,OAARX,OAAS,qBAAIvC,QAAQQ,MAAM,GAAG;4BAAUuB,KAAKoB,KAAK;;;;oBAAhEL,OAAQ;oBACd,+FAA+F;oBAC/F,yFAAyF;oBACzF,IAAIA,KAAKxB,IAAI,CAAC,SAAC8B;+BAAMA,EAAEC,IAAI,KAAK;wBAAO;wBACrC,MAAM,IAAIL,oBAAU,CAAC,qBAAqB;oBAC5C;oBACMD,UAAU/C,QAAQQ,MAAM,KAAK,IAAIsC,OAAOA,KAAKpC,MAAM,CAAC,SAAC4C;+BAAQtD,QAAQuD,KAAK,CAAC,SAAClD;mCAAWnB,gBAAgBoC,IAAI,CAAC,SAACkC;oCAA8BF;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAIjD;;;;oBACnK;;wBAAO0C,QAAQW,KAAK,CAAC,GAAG3B,KAAKoB,KAAK,EAAExC,GAAG,CAAC;gCAAG0C,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAChE"}