sensemaking 0.23.0 → 0.24.0

package/dist/cjs/store/turso/sql-functions.js

@@ -0,0 +1,174 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "rewriteFunctions", {
+    enumerable: true,
+    get: function() {
+        return rewriteFunctions;
+    }
+});
+var _errorsts = require("../../errors.js");
+// Turso can't register UDFs: has()/basename() are rewritten into portable SQL below; segment()
+// (Intl.Segmenter grapheme clustering, types.ts's 'segment' capability) has none, so it's rejected like an unsupported FTS5 operator.
+// Non-global: recursion (readArgs' contents rewritten via rewriteFunctions) reenters this module's
+// regexes before the outer scan resumes, and a global regex's lastIndex does not nest across that.
+var CALL_ANYWHERE = /\b(has|basename|segment)\s*\(/i;
+var CALL_AT_START = /^(has|basename|segment)\s*\(/i;
+function isWordChar(ch) {
+    return ch !== undefined && /[A-Za-z0-9_]/.test(ch);
+}
+// The index just past the quoted literal or SQL comment starting at i, or null if i starts
+// neither. Shared by the scan and readArgs so both leave quoted/comment text opaque to a call match.
+function skipOpaque(sql, i) {
+    var ch = sql[i];
+    if (ch === "'" || ch === '"') {
+        var j = i + 1;
+        while(j < sql.length){
+            if (sql[j] === ch) {
+                if (sql[j + 1] === ch) {
+                    j += 2;
+                    continue;
+                }
+                return j + 1;
+            }
+            j++;
+        }
+        return j; // unterminated literal: consume to end rather than loop forever
+    }
+    if (ch === '-' && sql[i + 1] === '-') {
+        var nl = sql.indexOf('\n', i);
+        return nl === -1 ? sql.length : nl + 1;
+    }
+    if (ch === '/' && sql[i + 1] === '*') {
+        var close = sql.indexOf('*/', i + 2);
+        return close === -1 ? sql.length : close + 2;
+    }
+    return null;
+}
+// `?N`/named placeholders bind by number/name, so repeating one is safe; an anonymous `?` takes
+// the next slot per occurrence, so it's numbered here first -- sqlHas/sqlBasename duplicate their arguments.
+function numberPlaceholders(sql) {
+    var out = '';
+    var i = 0;
+    var n = 0;
+    while(i < sql.length){
+        var _sql_;
+        var skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            out += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        if (sql[i] === '?' && !/[0-9]/.test((_sql_ = sql[i + 1]) !== null && _sql_ !== void 0 ? _sql_ : '')) {
+            n++;
+            out += "?".concat(n);
+            i++;
+            continue;
+        }
+        out += sql[i];
+        i++;
+    }
+    return out;
+}
+// The un-rewritten argument texts of the call whose '(' sits at parenStart, and the index just
+// past its matching ')'. Paren depth is tracked so a nested call's commas/parens aren't mistaken for ours.
+function readArgs(sql, parenStart) {
+    var args = [];
+    var depth = 1;
+    var cur = '';
+    var i = parenStart + 1;
+    while(i < sql.length){
+        var skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            cur += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        var ch = sql[i];
+        if (ch === '(') {
+            depth++;
+            cur += ch;
+            i++;
+            continue;
+        }
+        if (ch === ')') {
+            depth--;
+            i++;
+            if (depth === 0) break;
+            cur += ch;
+            continue;
+        }
+        if (ch === ',' && depth === 1) {
+            args.push(cur.trim());
+            cur = '';
+            i++;
+            continue;
+        }
+        cur += ch;
+        i++;
+    }
+    if (depth !== 0) throw new Error('turso: unterminated argument list starting at "'.concat(sql.slice(parenStart, parenStart + 30), '..."'));
+    if (cur.trim() !== '') args.push(cur.trim());
+    return {
+        args: args,
+        end: i
+    };
+}
+// field.startsWith('[') && JSON.parse(field) is an array, matched by membership; every other
+// field, stringified, matched by substring (sql-functions.ts's hasImpl, mirrored branch for branch).
+function sqlHas(fieldExpr, valueExpr) {
+    var isArray = "substr(".concat(fieldExpr, ",1,1) = '[' AND json_valid(").concat(fieldExpr, ")");
+    // json_each's argument is re-guarded by its own CASE (falling back to '[]'): turso evaluates a
+    // CASE branch's table function even when untaken, and a non-array argument there raises rather than yielding 0 rows (spike-verified).
+    return "(CASE\n    WHEN ".concat(fieldExpr, " IS NULL THEN 0\n    WHEN ").concat(isArray, " THEN\n      (SELECT COUNT(*) FROM json_each(CASE WHEN ").concat(isArray, " THEN ").concat(fieldExpr, " ELSE '[]' END) je WHERE CAST(je.value AS TEXT) = CAST(").concat(valueExpr, " AS TEXT)) > 0\n    ELSE instr(CAST(").concat(fieldExpr, " AS TEXT), CAST(").concat(valueExpr, " AS TEXT)) > 0\n  END)");
+}
+// posix.basename(String(path)), suffix stripped Unix-style (sql-functions.ts's basenameImpl). No
+// last-'/' primitive exists, so reverse() finds the first '/' from the end, then reverse() restores order.
+function sqlBasename(pathExpr, suffixExpr) {
+    var trimmed = "rtrim(CAST(".concat(pathExpr, " AS TEXT), '/')");
+    var name = "CASE\n      WHEN ".concat(trimmed, " = '' THEN ''\n      WHEN instr(").concat(trimmed, ", '/') = 0 THEN ").concat(trimmed, "\n      ELSE reverse(substr(reverse(").concat(trimmed, "), 1, instr(reverse(").concat(trimmed, "), '/') - 1))\n    END");
+    return "(CASE WHEN ".concat(pathExpr, " IS NULL THEN NULL ELSE\n    (CASE WHEN ").concat(suffixExpr, " IS NOT NULL AND CAST(").concat(suffixExpr, " AS TEXT) != '' AND CAST(").concat(suffixExpr, " AS TEXT) != (").concat(name, ")\n          AND substr((").concat(name, "), -length(CAST(").concat(suffixExpr, " AS TEXT))) = CAST(").concat(suffixExpr, " AS TEXT)\n     THEN substr((").concat(name, "), 1, length((").concat(name, ")) - length(CAST(").concat(suffixExpr, " AS TEXT)))\n     ELSE (").concat(name, ") END)\n  END)");
+}
+function rewriteFunctions(sql) {
+    if (!CALL_ANYWHERE.test(sql)) return sql;
+    // Numbered once, over the whole statement, before any argument is duplicated below -- rewrite()
+    // recurses over already-numbered substrings, never renumbering them a second time.
+    return rewrite(numberPlaceholders(sql));
+}
+function rewrite(sql) {
+    var out = '';
+    var i = 0;
+    while(i < sql.length){
+        var skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            out += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        var m = !isWordChar(sql[i - 1]) ? CALL_AT_START.exec(sql.slice(i)) : null;
+        if (m) {
+            var name = m[1].toLowerCase();
+            var parenStart = i + m[0].length - 1;
+            if (name === 'segment') {
+                throw new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "turso" does not implement segment() as a SQL function in this build (its client cannot register UDFs); rephrase the query without it, or set "store" to "sqlite" in this tree\'s config to run it as written');
+            }
+            var _readArgs = readArgs(sql, parenStart), args = _readArgs.args, end = _readArgs.end;
+            var rewrittenArgs = args.map(rewrite);
+            if (name === 'has') {
+                if (rewrittenArgs.length !== 2) throw new Error("turso: has() takes exactly 2 arguments (field, value), got ".concat(rewrittenArgs.length));
+                out += sqlHas(rewrittenArgs[0], rewrittenArgs[1]);
+            } else {
+                var _rewrittenArgs_;
+                if (rewrittenArgs.length < 1 || rewrittenArgs.length > 2) throw new Error("turso: basename() takes 1 or 2 arguments (path, suffix?), got ".concat(rewrittenArgs.length));
+                out += sqlBasename(rewrittenArgs[0], (_rewrittenArgs_ = rewrittenArgs[1]) !== null && _rewrittenArgs_ !== void 0 ? _rewrittenArgs_ : 'NULL');
+            }
+            i = end;
+            continue;
+        }
+        out += sql[i];
+        i++;
+    }
+    return out;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/turso/sql-functions.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/sql-functions.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\n\n// Turso can't register UDFs: has()/basename() are rewritten into portable SQL below; segment()\n// (Intl.Segmenter grapheme clustering, types.ts's 'segment' capability) has none, so it's rejected like an unsupported FTS5 operator.\n\n// Non-global: recursion (readArgs' contents rewritten via rewriteFunctions) reenters this module's\n// regexes before the outer scan resumes, and a global regex's lastIndex does not nest across that.\nconst CALL_ANYWHERE = /\\b(has|basename|segment)\\s*\\(/i;\nconst CALL_AT_START = /^(has|basename|segment)\\s*\\(/i;\n\nfunction isWordChar(ch: string | undefined): boolean {\n  return ch !== undefined && /[A-Za-z0-9_]/.test(ch);\n}\n\n// The index just past the quoted literal or SQL comment starting at i, or null if i starts\n// neither. Shared by the scan and readArgs so both leave quoted/comment text opaque to a call match.\nfunction skipOpaque(sql: string, i: number): number | null {\n  const ch = sql[i];\n  if (ch === \"'\" || ch === '\"') {\n    let j = i + 1;\n    while (j < sql.length) {\n      if (sql[j] === ch) {\n        if (sql[j + 1] === ch) {\n          j += 2;\n          continue;\n        }\n        return j + 1;\n      }\n      j++;\n    }\n    return j; // unterminated literal: consume to end rather than loop forever\n  }\n  if (ch === '-' && sql[i + 1] === '-') {\n    const nl = sql.indexOf('\\n', i);\n    return nl === -1 ? sql.length : nl + 1;\n  }\n  if (ch === '/' && sql[i + 1] === '*') {\n    const close = sql.indexOf('*/', i + 2);\n    return close === -1 ? sql.length : close + 2;\n  }\n  return null;\n}\n\n// `?N`/named placeholders bind by number/name, so repeating one is safe; an anonymous `?` takes\n// the next slot per occurrence, so it's numbered here first -- sqlHas/sqlBasename duplicate their arguments.\nfunction numberPlaceholders(sql: string): string {\n  let out = '';\n  let i = 0;\n  let n = 0;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      out += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    if (sql[i] === '?' && !/[0-9]/.test(sql[i + 1] ?? '')) {\n      n++;\n      out += `?${n}`;\n      i++;\n      continue;\n    }\n    out += sql[i];\n    i++;\n  }\n  return out;\n}\n\n// The un-rewritten argument texts of the call whose '(' sits at parenStart, and the index just\n// past its matching ')'. Paren depth is tracked so a nested call's commas/parens aren't mistaken for ours.\nfunction readArgs(sql: string, parenStart: number): { args: string[]; end: number } {\n  const args: string[] = [];\n  let depth = 1;\n  let cur = '';\n  let i = parenStart + 1;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      cur += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    const ch = sql[i];\n    if (ch === '(') {\n      depth++;\n      cur += ch;\n      i++;\n      continue;\n    }\n    if (ch === ')') {\n      depth--;\n      i++;\n      if (depth === 0) break;\n      cur += ch;\n      continue;\n    }\n    if (ch === ',' && depth === 1) {\n      args.push(cur.trim());\n      cur = '';\n      i++;\n      continue;\n    }\n    cur += ch;\n    i++;\n  }\n  if (depth !== 0) throw new Error(`turso: unterminated argument list starting at \"${sql.slice(parenStart, parenStart + 30)}...\"`);\n  if (cur.trim() !== '') args.push(cur.trim());\n  return { args, end: i };\n}\n\n// field.startsWith('[') && JSON.parse(field) is an array, matched by membership; every other\n// field, stringified, matched by substring (sql-functions.ts's hasImpl, mirrored branch for branch).\nfunction sqlHas(fieldExpr: string, valueExpr: string): string {\n  const isArray = `substr(${fieldExpr},1,1) = '[' AND json_valid(${fieldExpr})`;\n  // json_each's argument is re-guarded by its own CASE (falling back to '[]'): turso evaluates a\n  // CASE branch's table function even when untaken, and a non-array argument there raises rather than yielding 0 rows (spike-verified).\n  return `(CASE\n    WHEN ${fieldExpr} IS NULL THEN 0\n    WHEN ${isArray} THEN\n      (SELECT COUNT(*) FROM json_each(CASE WHEN ${isArray} THEN ${fieldExpr} ELSE '[]' END) je WHERE CAST(je.value AS TEXT) = CAST(${valueExpr} AS TEXT)) > 0\n    ELSE instr(CAST(${fieldExpr} AS TEXT), CAST(${valueExpr} AS TEXT)) > 0\n  END)`;\n}\n\n// posix.basename(String(path)), suffix stripped Unix-style (sql-functions.ts's basenameImpl). No\n// last-'/' primitive exists, so reverse() finds the first '/' from the end, then reverse() restores order.\nfunction sqlBasename(pathExpr: string, suffixExpr: string): string {\n  const trimmed = `rtrim(CAST(${pathExpr} AS TEXT), '/')`;\n  const name = `CASE\n      WHEN ${trimmed} = '' THEN ''\n      WHEN instr(${trimmed}, '/') = 0 THEN ${trimmed}\n      ELSE reverse(substr(reverse(${trimmed}), 1, instr(reverse(${trimmed}), '/') - 1))\n    END`;\n  return `(CASE WHEN ${pathExpr} IS NULL THEN NULL ELSE\n    (CASE WHEN ${suffixExpr} IS NOT NULL AND CAST(${suffixExpr} AS TEXT) != '' AND CAST(${suffixExpr} AS TEXT) != (${name})\n          AND substr((${name}), -length(CAST(${suffixExpr} AS TEXT))) = CAST(${suffixExpr} AS TEXT)\n     THEN substr((${name}), 1, length((${name})) - length(CAST(${suffixExpr} AS TEXT)))\n     ELSE (${name}) END)\n  END)`;\n}\n\n// Rewrites every has()/basename() call into the portable SQL above, recursing into each call's own\n// arguments first so a nested `instr(basename(f.path), 'x')` resolves inside out; segment() instead raises STORE_CAPABILITY_MISSING, named, like an unsupported FTS5 operator.\nexport function rewriteFunctions(sql: string): string {\n  if (!CALL_ANYWHERE.test(sql)) return sql;\n  // Numbered once, over the whole statement, before any argument is duplicated below -- rewrite()\n  // recurses over already-numbered substrings, never renumbering them a second time.\n  return rewrite(numberPlaceholders(sql));\n}\n\nfunction rewrite(sql: string): string {\n  let out = '';\n  let i = 0;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      out += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    const m = !isWordChar(sql[i - 1]) ? CALL_AT_START.exec(sql.slice(i)) : null;\n    if (m) {\n      const name = m[1].toLowerCase();\n      const parenStart = i + m[0].length - 1;\n      if (name === 'segment') {\n        throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement segment() as a SQL function in this build (its client cannot register UDFs); rephrase the query without it, or set \"store\" to \"sqlite\" in this tree's config to run it as written`);\n      }\n      const { args, end } = readArgs(sql, parenStart);\n      const rewrittenArgs = args.map(rewrite);\n      if (name === 'has') {\n        if (rewrittenArgs.length !== 2) throw new Error(`turso: has() takes exactly 2 arguments (field, value), got ${rewrittenArgs.length}`);\n        out += sqlHas(rewrittenArgs[0], rewrittenArgs[1]);\n      } else {\n        if (rewrittenArgs.length < 1 || rewrittenArgs.length > 2) throw new Error(`turso: basename() takes 1 or 2 arguments (path, suffix?), got ${rewrittenArgs.length}`);\n        out += sqlBasename(rewrittenArgs[0], rewrittenArgs[1] ?? 'NULL');\n      }\n      i = end;\n      continue;\n    }\n    out += sql[i];\n    i++;\n  }\n  return out;\n}\n"],"names":["rewriteFunctions","CALL_ANYWHERE","CALL_AT_START","isWordChar","ch","undefined","test","skipOpaque","sql","i","j","length","nl","indexOf","close","numberPlaceholders","out","n","skip","slice","readArgs","parenStart","args","depth","cur","push","trim","Error","end","sqlHas","fieldExpr","valueExpr","isArray","sqlBasename","pathExpr","suffixExpr","trimmed","name","rewrite","m","exec","toLowerCase","SenseError","rewrittenArgs","map"],"mappings":";;;;+BA+IgBA;;;eAAAA;;;wBA/IW;AAE3B,+FAA+F;AAC/F,sIAAsI;AAEtI,mGAAmG;AACnG,mGAAmG;AACnG,IAAMC,gBAAgB;AACtB,IAAMC,gBAAgB;AAEtB,SAASC,WAAWC,EAAsB;IACxC,OAAOA,OAAOC,aAAa,eAAeC,IAAI,CAACF;AACjD;AAEA,2FAA2F;AAC3F,qGAAqG;AACrG,SAASG,WAAWC,GAAW,EAAEC,CAAS;IACxC,IAAML,KAAKI,GAAG,CAACC,EAAE;IACjB,IAAIL,OAAO,OAAOA,OAAO,KAAK;QAC5B,IAAIM,IAAID,IAAI;QACZ,MAAOC,IAAIF,IAAIG,MAAM,CAAE;YACrB,IAAIH,GAAG,CAACE,EAAE,KAAKN,IAAI;gBACjB,IAAII,GAAG,CAACE,IAAI,EAAE,KAAKN,IAAI;oBACrBM,KAAK;oBACL;gBACF;gBACA,OAAOA,IAAI;YACb;YACAA;QACF;QACA,OAAOA,GAAG,gEAAgE;IAC5E;IACA,IAAIN,OAAO,OAAOI,GAAG,CAACC,IAAI,EAAE,KAAK,KAAK;QACpC,IAAMG,KAAKJ,IAAIK,OAAO,CAAC,MAAMJ;QAC7B,OAAOG,OAAO,CAAC,IAAIJ,IAAIG,MAAM,GAAGC,KAAK;IACvC;IACA,IAAIR,OAAO,OAAOI,GAAG,CAACC,IAAI,EAAE,KAAK,KAAK;QACpC,IAAMK,QAAQN,IAAIK,OAAO,CAAC,MAAMJ,IAAI;QACpC,OAAOK,UAAU,CAAC,IAAIN,IAAIG,MAAM,GAAGG,QAAQ;IAC7C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,6GAA6G;AAC7G,SAASC,mBAAmBP,GAAW;IACrC,IAAIQ,MAAM;IACV,IAAIP,IAAI;IACR,IAAIQ,IAAI;IACR,MAAOR,IAAID,IAAIG,MAAM,CAAE;YAOeH;QANpC,IAAMU,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBF,OAAOR,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,IAAIV,GAAG,CAACC,EAAE,KAAK,OAAO,CAAC,QAAQH,IAAI,EAACE,QAAAA,GAAG,CAACC,IAAI,EAAE,cAAVD,mBAAAA,QAAc,KAAK;YACrDS;YACAD,OAAO,AAAC,IAAK,OAAFC;YACXR;YACA;QACF;QACAO,OAAOR,GAAG,CAACC,EAAE;QACbA;IACF;IACA,OAAOO;AACT;AAEA,+FAA+F;AAC/F,2GAA2G;AAC3G,SAASI,SAASZ,GAAW,EAAEa,UAAkB;IAC/C,IAAMC,OAAiB,EAAE;IACzB,IAAIC,QAAQ;IACZ,IAAIC,MAAM;IACV,IAAIf,IAAIY,aAAa;IACrB,MAAOZ,IAAID,IAAIG,MAAM,CAAE;QACrB,IAAMO,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBM,OAAOhB,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,IAAMd,KAAKI,GAAG,CAACC,EAAE;QACjB,IAAIL,OAAO,KAAK;YACdmB;YACAC,OAAOpB;YACPK;YACA;QACF;QACA,IAAIL,OAAO,KAAK;YACdmB;YACAd;YACA,IAAIc,UAAU,GAAG;YACjBC,OAAOpB;YACP;QACF;QACA,IAAIA,OAAO,OAAOmB,UAAU,GAAG;YAC7BD,KAAKG,IAAI,CAACD,IAAIE,IAAI;YAClBF,MAAM;YACNf;YACA;QACF;QACAe,OAAOpB;QACPK;IACF;IACA,IAAIc,UAAU,GAAG,MAAM,IAAII,MAAM,AAAC,kDAAwF,OAAvCnB,IAAIW,KAAK,CAACE,YAAYA,aAAa,KAAI;IAC1H,IAAIG,IAAIE,IAAI,OAAO,IAAIJ,KAAKG,IAAI,CAACD,IAAIE,IAAI;IACzC,OAAO;QAAEJ,MAAAA;QAAMM,KAAKnB;IAAE;AACxB;AAEA,6FAA6F;AAC7F,qGAAqG;AACrG,SAASoB,OAAOC,SAAiB,EAAEC,SAAiB;IAClD,IAAMC,UAAU,AAAC,UAAgDF,OAAvCA,WAAU,+BAAuC,OAAVA,WAAU;IAC3E,+FAA+F;IAC/F,sIAAsI;IACtI,OAAO,AAAC,mBAECE,OADAF,WAAU,8BAE6BE,OADvCA,SAAQ,2DAC+CF,OAAhBE,SAAQ,UAA2ED,OAAnED,WAAU,2DACtDA,OAD+GC,WAAU,wCAC7FA,OAA5BD,WAAU,oBAA4B,OAAVC,WAAU;AAE5D;AAEA,iGAAiG;AACjG,2GAA2G;AAC3G,SAASE,YAAYC,QAAgB,EAAEC,UAAkB;IACvD,IAAMC,UAAU,AAAC,cAAsB,OAATF,UAAS;IACvC,IAAMG,OAAO,AAAC,oBAEGD,OADNA,SAAQ,oCACwBA,OAA1BA,SAAQ,oBACSA,OADSA,SAAQ,wCACaA,OAA9BA,SAAQ,wBAA8B,OAARA,SAAQ;IAExE,OAAO,AAAC,cACOD,OADMD,UAAS,4CACoBC,OAAnCA,YAAW,0BAA8DA,OAAtCA,YAAW,6BAAsDE,OAA3BF,YAAW,kBAC7EE,OAD6FA,MAAK,6BAC3EF,OAAvBE,MAAK,oBAAkDF,OAAhCA,YAAW,uBACtCE,OAD2DF,YAAW,iCACjDE,OAArBA,MAAK,kBAAwCF,OAAxBE,MAAK,qBACjCA,OADoDF,YAAW,4BAC1D,OAALE,MAAK;AAElB;AAIO,SAASrC,iBAAiBQ,GAAW;IAC1C,IAAI,CAACP,cAAcK,IAAI,CAACE,MAAM,OAAOA;IACrC,gGAAgG;IAChG,mFAAmF;IACnF,OAAO8B,QAAQvB,mBAAmBP;AACpC;AAEA,SAAS8B,QAAQ9B,GAAW;IAC1B,IAAIQ,MAAM;IACV,IAAIP,IAAI;IACR,MAAOA,IAAID,IAAIG,MAAM,CAAE;QACrB,IAAMO,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBF,OAAOR,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,IAAMqB,IAAI,CAACpC,WAAWK,GAAG,CAACC,IAAI,EAAE,IAAIP,cAAcsC,IAAI,CAAChC,IAAIW,KAAK,CAACV,MAAM;QACvE,IAAI8B,GAAG;YACL,IAAMF,OAAOE,CAAC,CAAC,EAAE,CAACE,WAAW;YAC7B,IAAMpB,aAAaZ,IAAI8B,CAAC,CAAC,EAAE,CAAC5B,MAAM,GAAG;YACrC,IAAI0B,SAAS,WAAW;gBACtB,MAAM,IAAIK,oBAAU,CAAC,4BAA4B;YACnD;YACA,IAAsBtB,YAAAA,SAASZ,KAAKa,aAA5BC,OAAcF,UAAdE,MAAMM,MAAQR,UAARQ;YACd,IAAMe,gBAAgBrB,KAAKsB,GAAG,CAACN;YAC/B,IAAID,SAAS,OAAO;gBAClB,IAAIM,cAAchC,MAAM,KAAK,GAAG,MAAM,IAAIgB,MAAM,AAAC,8DAAkF,OAArBgB,cAAchC,MAAM;gBAClIK,OAAOa,OAAOc,aAAa,CAAC,EAAE,EAAEA,aAAa,CAAC,EAAE;YAClD,OAAO;oBAEgCA;gBADrC,IAAIA,cAAchC,MAAM,GAAG,KAAKgC,cAAchC,MAAM,GAAG,GAAG,MAAM,IAAIgB,MAAM,AAAC,iEAAqF,OAArBgB,cAAchC,MAAM;gBAC/JK,OAAOiB,YAAYU,aAAa,CAAC,EAAE,GAAEA,kBAAAA,aAAa,CAAC,EAAE,cAAhBA,6BAAAA,kBAAoB;YAC3D;YACAlC,IAAImB;YACJ;QACF;QACAZ,OAAOR,GAAG,CAACC,EAAE;QACbA;IACF;IACA,OAAOO;AACT"}

package/dist/cjs/store/turso/store.js

@@ -23,6 +23,7 @@ var _vectorsts = require("../vectors.js");
 var _connectionts = require("./connection.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
+var _sqlfunctionsts = require("./sql-functions.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -363,10 +364,11 @@ function _wrap_async_generator(fn) {
     };
 }
 var CAPABILITIES = new Set([
-    'lexical',
-    'phrases',
     'vectors'
 ]);
+// close() reclaims once the cache file outgrows its compact size by this factor: the disk
+// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.
+var BLOAT_FACTOR = 1.5;
 function createStore(db, conn) {
     return {
         name: 'turso',
@@ -512,7 +514,7 @@ function createStore(db, conn) {
                             case 0:
                                 return [
                                     4,
-                                    db.prepare(sql)
+                                    db.prepare((0, _sqlfunctionsts.rewriteFunctions)(sql))
                                 ];
                             case 1:
                                 stmt = _state.sent();
@@ -555,21 +557,120 @@ function createStore(db, conn) {
         },
         close: function close() {
             return _async_to_generator(function() {
+                var reclaimPath, _ref, path, base, size, _tmp, err;
                 return _ts_generator(this, function(_state) {
                     switch(_state.label){
                         case 0:
+                            // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.
                             return [
                                 4,
                                 (0, _connectionts.checkpointWal)(db)
                             ];
                         case 1:
                             _state.sent();
+                            reclaimPath = null;
+                            _state.label = 2;
+                        case 2:
+                            _state.trys.push([
+                                2,
+                                12,
+                                ,
+                                13
+                            ]);
+                            return [
+                                4,
+                                (0, _connectionts.cacheFilePath)(conn)
+                            ];
+                        case 3:
+                            path = _state.sent();
+                            return [
+                                4,
+                                (0, _sharedts.getMeta)(conn, 'compact_size')
+                            ];
+                        case 4:
+                            base = Number.apply(void 0, [
+                                (_ref = _state.sent()) !== null && _ref !== void 0 ? _ref : '0'
+                            ]);
+                            if (!(path === null)) return [
+                                3,
+                                5
+                            ];
+                            _tmp = null;
+                            return [
+                                3,
+                                7
+                            ];
+                        case 5:
+                            return [
+                                4,
+                                (0, _connectionts.fileSize)(path)
+                            ];
+                        case 6:
+                            _tmp = _state.sent();
+                            _state.label = 7;
+                        case 7:
+                            size = _tmp;
+                            if (!(size !== null)) return [
+                                3,
+                                11
+                            ];
+                            if (!(base === 0)) return [
+                                3,
+                                10
+                            ];
+                            return [
+                                4,
+                                (0, _sharedts.setMeta)(conn, 'compact_size', String(size))
+                            ];
+                        case 8:
+                            _state.sent();
+                            // The meta write landed past the checkpoint; leave no WAL for the next opener.
+                            return [
+                                4,
+                                (0, _connectionts.checkpointWal)(db)
+                            ];
+                        case 9:
+                            _state.sent();
+                            return [
+                                3,
+                                11
+                            ];
+                        case 10:
+                            if (size > BLOAT_FACTOR * base) {
+                                reclaimPath = path;
+                            }
+                            _state.label = 11;
+                        case 11:
+                            return [
+                                3,
+                                13
+                            ];
+                        case 12:
+                            err = _state.sent();
+                            console.error("sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ".concat(err.message));
+                            return [
+                                3,
+                                13
+                            ];
+                        case 13:
                             return [
                                 4,
                                 db.close()
                             ];
-                        case 2:
+                        case 14:
+                            _state.sent();
+                            if (!(reclaimPath !== null)) return [
+                                3,
+                                16
+                            ];
+                            return [
+                                4,
+                                (0, _connectionts.reclaimSpace)(reclaimPath)
+                            ];
+                        case 15:
                             _state.sent();
+                            _state.label = 16;
+                        case 16:
                             return [
                                 2
                             ];

package/dist/cjs/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { checkpointWal } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await checkpointWal(db);\n      await db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close","checkpointWal"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACZ;4BACH;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACD;;;gCAAxBwC,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;;;4BACJ;;gCAAMC,IAAAA,2BAAa,EAAClD;;;4BAApB;4BACA;;gCAAMA,GAAGiD,KAAK;;;4BAAd;;;;;;YACF;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns, getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'segment': has()/basename() resolve via SQL rewrite below (sql-functions.ts), but segment() has no SQL form and the client cannot register it as a UDF.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors']);\n\n// close() reclaims once the cache file outgrows its compact size by this factor: the disk\n// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.\nconst BLOAT_FACTOR = 1.5;\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(rewriteFunctions(sql));\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.\n      await checkpointWal(db);\n      let reclaimPath: string | null = null;\n      try {\n        const path = await cacheFilePath(conn);\n        const base = Number((await getMeta(conn, 'compact_size')) ?? '0');\n        const size = path === null ? null : await fileSize(path);\n        if (size !== null) {\n          // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a\n          // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.\n          if (base === 0) {\n            await setMeta(conn, 'compact_size', String(size));\n            // The meta write landed past the checkpoint; leave no WAL for the next opener.\n            await checkpointWal(db);\n          } else if (size > BLOAT_FACTOR * base) {\n            reclaimPath = path;\n          }\n        }\n      } catch (err) {\n        console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${(err as Error).message}`);\n      }\n      await db.close();\n      // After this store's connection is closed, never before: turso hands the file to one\n      // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).\n      if (reclaimPath !== null) await reclaimSpace(reclaimPath);\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","BLOAT_FACTOR","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","rewriteFunctions","safeIntegers","iterate","params","close","reclaimPath","base","size","err","checkpointWal","cacheFilePath","getMeta","Number","fileSize","setMeta","String","console","error","message","reclaimSpace"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAQGC;eAAAA;;;uBApBW;wBACkB;6BACb;yBAEU;4BAC2B;4BAC1C;yBACE;8BACI;0BAC6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;CAAU;AAExE,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,eAAe;AAId,SAASF,YAAYG,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcP;QACRQ,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACwC,IAAAA,gCAAgB,EAACzC;;;gCAAzCwC,OAAO;gCACbA,KAAKE,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLlC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DmC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCJ;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKG,OAAO,OAAZH,OAAa,qBAAGI;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;oBAGAC,aAGmB,MADff,MACAgB,MACAC,YAYCC;;;;4BAlBT,4FAA4F;4BAC5F;;gCAAMC,IAAAA,2BAAa,EAACvD;;;4BAApB;4BACImD,cAA6B;;;;;;;;;4BAElB;;gCAAMK,IAAAA,2BAAa,EAACvD;;;4BAA3BmC,OAAO;4BACQ;;gCAAMqB,IAAAA,iBAAO,EAACxD,MAAM;;;4BAAnCmD,OAAOM;iCAAQ,OAAA,2BAAA,kBAAA,OAAwC;;iCAChDtB,CAAAA,SAAS,IAAG,GAAZA;;;;mCAAgB;;;;;;4BAAO;;gCAAMuB,IAAAA,sBAAQ,EAACvB;;;mCAAf;;;4BAA9BiB;iCACFA,CAAAA,SAAS,IAAG,GAAZA;;;;iCAGED,CAAAA,SAAS,CAAA,GAATA;;;;4BACF;;gCAAMQ,IAAAA,iBAAO,EAAC3D,MAAM,gBAAgB4D,OAAOR;;;4BAA3C;4BACA,+EAA+E;4BAC/E;;gCAAME,IAAAA,2BAAa,EAACvD;;;4BAApB;;;;;;4BACK,IAAIqD,OAAOtD,eAAeqD,MAAM;gCACrCD,cAAcf;4BAChB;;;;;;;;4BAEKkB;4BACPQ,QAAQC,KAAK,CAAC,AAAC,4FAAkH,OAAvB,AAACT,IAAcU,OAAO;;;;;;4BAElI;;gCAAMhE,GAAGkD,KAAK;;;4BAAd;iCAGIC,CAAAA,gBAAgB,IAAG,GAAnBA;;;;4BAAsB;;gCAAMc,IAAAA,0BAAY,EAACd;;;4BAAnB;;;;;;;;YAC5B;;IACF;AACF"}

package/dist/cjs/store/turso/vectors.js

@@ -57,6 +57,17 @@ function _async_to_generator(fn) {
         });
     };
 }
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
 function _iterable_to_array(iter) {
     if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
         return Array.from(iter);
@@ -92,6 +103,21 @@ function _non_iterable_rest() {
 function _non_iterable_spread() {
     throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
 }
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
 function _sliced_to_array(arr, i) {
     return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
 }
@@ -277,7 +303,7 @@ function scanCandidates(conn, qv, dims, fetch, allowed) {
                     q = padded(qv, dims);
                     return [
                         4,
-                        conn.prepare('SELECT "path", start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL')
+                        conn.prepare('SELECT "path", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY "path", chunk')
                     ];
                 case 1:
                     stmt = _state.sent();
@@ -295,10 +321,13 @@ function scanCandidates(conn, qv, dims, fetch, allowed) {
                             if (allowed && !allowed.has(row.path)) continue;
                             score = cosineSimilarity(q, decode(row.vector, dims));
                             existing = best.get(row.path);
-                            if (!existing || score > existing.score) best.set(row.path, {
-                                score: score,
-                                lines: "L".concat(row.start_line, "-").concat(row.end_line)
-                            });
+                            if (!existing || score > existing.score || score === existing.score && row.chunk < existing.chunk) {
+                                best.set(row.path, {
+                                    score: score,
+                                    chunk: row.chunk,
+                                    lines: "L".concat(row.start_line, "-").concat(row.end_line)
+                                });
+                            }
                         }
                     } catch (err) {
                         _didIteratorError = true;
@@ -316,14 +345,17 @@ function scanCandidates(conn, qv, dims, fetch, allowed) {
                     }
                     return [
                         2,
-                        _to_consumable_array(best.entries()).sort(function(a, b) {
-                            return b[1].score - a[1].score;
-                        }).slice(0, fetch).map(function(param) {
-                            var _param = _sliced_to_array(param, 2), path = _param[0], b = _param[1];
+                        _to_consumable_array(best.entries()).map(function(param) {
+                            var _param = _sliced_to_array(param, 2), path = _param[0], value = _param[1];
+                            return _object_spread({
+                                path: path
+                            }, value);
+                        }).sort(_vectorsts.compareVectorScores).slice(0, fetch).map(function(param) {
+                            var path = param.path, lines = param.lines, score = param.score;
                             return {
                                 path: path,
-                                lines: b.lines,
-                                similarity: (0, _vectorsts.asCosine)(b.score)
+                                lines: lines,
+                                similarity: (0, _vectorsts.asCosine)(score)
                             };
                         })
                     ];
@@ -418,10 +450,14 @@ function scanSimilar(conn, dims, path, opts) {
                     }
                     return [
                         2,
-                        _to_consumable_array(best.entries()).sort(function(a, b) {
-                            return b[1] - a[1];
-                        }).slice(0, opts.k).map(function(param) {
+                        _to_consumable_array(best.entries()).map(function(param) {
                             var _param = _sliced_to_array(param, 2), p = _param[0], score = _param[1];
+                            return {
+                                path: p,
+                                score: score
+                            };
+                        }).sort(_vectorsts.compareVectorScores).slice(0, opts.k).map(function(param) {
+                            var p = param.path, score = param.score;
                             return {
                                 path: p,
                                 similarity: (0, _vectorsts.asCosine)(score)

package/dist/cjs/store/turso/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\n\n// Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans\n// score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.\n\n// A model narrower than the column zero-pads: the added dimensions contribute nothing to\n// either dot product or norm, so cosine is unchanged.\nfunction padded(values: ArrayLike<number>, dims: number): Float32Array {\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\n// A bound Float32Array stores truncated bytes with no error; a Buffer view over the same\n// bytes round-trips exact (measured 0.7.2).\nfunction toBlob(v: Float32Array): Buffer {\n  return Buffer.from(v.buffer, v.byteOffset, v.byteLength);\n}\n\nfunction decode(vector: Buffer, dims: number): Float32Array {\n  return new Float32Array(vector.buffer, vector.byteOffset, dims);\n}\n\n// Rows arrive int8-quantized with a per-vector scale (embed/query.ts's toStore); this store\n// dequantizes to floats, as duckdb does.\nfunction dequantize(row: VectorWriteRow, dims: number): Float32Array {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(q.length, dims); d++) out[d] = q[d] * row.scale;\n  return out;\n}\n\nfunction cosineSimilarity(a: Float32Array, b: Float32Array): number {\n  let dot = 0;\n  let na = 0;\n  let nb = 0;\n  for (let d = 0; d < a.length; d++) {\n    dot += a[d] * b[d];\n    na += a[d] * a[d];\n    nb += b[d] * b[d];\n  }\n  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-32);\n}\n\n// `scale` stays NULL: the column exists so the row shape matches the other stores, and\n// cosine is scale-invariant once the value is dequantized here.\nexport async function writeVectorBatch(conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [toBlob(dequantize(row, dims)), row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine. GROUP BY with MIN() cannot do this on 0.7.2: the aggregate\n// is right but the bare start_line/end_line come from an arbitrary row in the group.\nexport async function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const q = padded(qv, dims);\n  const stmt = await conn.prepare('SELECT \"path\", start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; start_line: number; end_line: number; vector: Buffer }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const score = cosineSimilarity(q, decode(row.vector, dims));\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured\n// 6-8x slower here at 26k chunks (2026-08-30).\nexport async function scanSimilar(conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Buffer }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => decode(row.vector, dims));\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Buffer }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = decode(row.vector, dims);\n    for (const t of targets) {\n      const score = cosineSimilarity(t, other);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","padded","values","dims","out","Float32Array","d","Math","min","length","toBlob","v","Buffer","from","buffer","byteOffset","byteLength","decode","vector","dequantize","row","q","Int8Array","scale","cosineSimilarity","a","b","dot","na","nb","sqrt","conn","rows","runBatch","map","path","chunk","qv","fetch","allowed","stmt","best","score","existing","prepare","all","Map","has","get","set","lines","start_line","end_line","entries","sort","slice","similarity","asCosine","opts","targetStmt","targetRows","targets","other","t","size","sampleEvenly","exclude","undefined","k","p"],"mappings":";;;;;;;;;;;QAyDsBA;eAAAA;;QAoBAC;eAAAA;;QA9BAC;eAAAA;;;yBA9CiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvC,0FAA0F;AAC1F,wFAAwF;AAExF,yFAAyF;AACzF,sDAAsD;AACtD,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,IAAMC,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOF;AACT;AAEA,yFAAyF;AACzF,4CAA4C;AAC5C,SAASM,OAAOC,CAAe;IAC7B,OAAOC,OAAOC,IAAI,CAACF,EAAEG,MAAM,EAAEH,EAAEI,UAAU,EAAEJ,EAAEK,UAAU;AACzD;AAEA,SAASC,OAAOC,MAAc,EAAEf,IAAY;IAC1C,OAAO,IAAIE,aAAaa,OAAOJ,MAAM,EAAEI,OAAOH,UAAU,EAAEZ;AAC5D;AAEA,4FAA4F;AAC5F,yCAAyC;AACzC,SAASgB,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,IAAMkB,IAAI,IAAIC,UAAUF,IAAIF,MAAM,CAACJ,MAAM,EAAEM,IAAIF,MAAM,CAACH,UAAU,EAAEK,IAAIF,MAAM,CAACF,UAAU;IACvF,IAAMZ,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACa,EAAEZ,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGe,CAAC,CAACf,EAAE,GAAGc,IAAIG,KAAK;IAC5E,OAAOnB;AACT;AAEA,SAASoB,iBAAiBC,CAAe,EAAEC,CAAe;IACxD,IAAIC,MAAM;IACV,IAAIC,KAAK;IACT,IAAIC,KAAK;IACT,IAAK,IAAIvB,IAAI,GAAGA,IAAImB,EAAEhB,MAAM,EAAEH,IAAK;QACjCqB,OAAOF,CAAC,CAACnB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;QAClBsB,MAAMH,CAAC,CAACnB,EAAE,GAAGmB,CAAC,CAACnB,EAAE;QACjBuB,MAAMH,CAAC,CAACpB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;IACnB;IACA,OAAOqB,MAAOpB,CAAAA,KAAKuB,IAAI,CAACF,MAAMrB,KAAKuB,IAAI,CAACD,MAAM,KAAI;AACpD;AAIO,SAAe7B,iBAAiB+B,IAAgB,EAAE5B,IAAY,EAAE6B,IAAsB;;;;;oBAC3F,IAAIA,KAAKvB,MAAM,KAAK,GAAG;;;oBACvB;;wBAAMsB,KAAKE,QAAQ,CACjB,mEACAD,KAAKE,GAAG,CAAC,SAACd;mCAAQ;gCAACV,OAAOS,WAAWC,KAAKjB;gCAAQiB,IAAIe,IAAI;gCAAEf,IAAIgB,KAAK;6BAAC;;;;oBAFxE;;;;;;IAIF;;AAIO,SAAetC,eAAeiC,IAAgB,EAAEM,EAAgB,EAAElC,IAAY,EAAEmC,KAAa,EAAEC,OAAqB;;YACnHlB,GACAmB,MACAR,MAEAS,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB,KAEHsB,OACAC;;;;oBARFtB,IAAIpB,OAAOoC,IAAIlC;oBACR;;wBAAM4B,KAAKa,OAAO,CAAC;;;oBAA1BJ,OAAO;oBACC;;wBAAMA,KAAKK,GAAG;;;oBAAtBb,OAAQ;oBAERS,OAAO,IAAIK;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAad,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbZ,MAAN;4BACH,IAAImB,WAAW,CAACA,QAAQQ,GAAG,CAAC3B,IAAIe,IAAI,GAAG;4BACjCO,QAAQlB,iBAAiBH,GAAGJ,OAAOG,IAAIF,MAAM,EAAEf;4BAC/CwC,WAAWF,KAAKO,GAAG,CAAC5B,IAAIe,IAAI;4BAClC,IAAI,CAACQ,YAAYD,QAAQC,SAASD,KAAK,EAAED,KAAKQ,GAAG,CAAC7B,IAAIe,IAAI,EAAE;gCAAEO,OAAAA;gCAAOQ,OAAO,AAAC,IAAqB9B,OAAlBA,IAAI+B,UAAU,EAAC,KAAgB,OAAb/B,IAAIgC,QAAQ;4BAAG;wBACnH;;wBALK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAML;;wBAAQ,qBAAGX,KAAKY,OAAO,IACpBC,IAAI,CAAC,SAAC7B,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACgB,KAAK,GAAGjB,CAAC,CAAC,EAAE,CAACiB,KAAK;2BACtCa,KAAK,CAAC,GAAGjB,OACTJ,GAAG,CAAC;qEAAEC,kBAAMT;mCAAQ;gCAAES,MAAAA;gCAAMe,OAAOxB,EAAEwB,KAAK;gCAAEM,YAAYC,IAAAA,mBAAQ,EAAC/B,EAAEgB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAAe3C,YAAYgC,IAAgB,EAAE5B,IAAY,EAAEgC,IAAY,EAAEuB,IAAgE;;YAExIC,YACAC,YAEAC,SAEArB,MACAR,MACAS,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB,KAEH0C,OACD,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,GACHrB,OACAC;;;;oBAdV,IAAIe,KAAKnB,OAAO,IAAImB,KAAKnB,OAAO,CAACyB,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAMjC,KAAKa,OAAO,CAAC;;;oBAAhCe,aAAa;oBACC;;wBAAMA,WAAWd,GAAG,CAACV;;;oBAAnCyB,aAAc;oBACpB,IAAIA,WAAWnD,MAAM,KAAK,GAAG;;;;oBACvBoD,UAAUI,IAAAA,uBAAY,EAACL,YAAY1B,GAAG,CAAC,SAACd;+BAAQH,OAAOG,IAAIF,MAAM,EAAEf;;oBAE5D;;wBAAM4B,KAAKa,OAAO,CAAC;;;oBAA1BJ,OAAO;oBACC;;wBAAMA,KAAKK,GAAG;;;oBAAtBb,OAAQ;oBACRS,OAAO,IAAIK;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAad,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbZ,MAAN;4BACH,IAAIA,IAAIe,IAAI,KAAKA,QAAQuB,KAAKQ,OAAO,CAACnB,GAAG,CAAC3B,IAAIe,IAAI,KAAMuB,KAAKnB,OAAO,IAAI,CAACmB,KAAKnB,OAAO,CAACQ,GAAG,CAAC3B,IAAIe,IAAI,GAAI;4BAChG2B,QAAQ7C,OAAOG,IAAIF,MAAM,EAAEf;4BAC5B,mCAAA,4BAAA;;gCAAL,IAAK,aAAW0D,8BAAX,8BAAA,SAAA,0BAAA,kCAAoB;oCAAdE,IAAN;oCACGrB,QAAQlB,iBAAiBuC,GAAGD;oCAC5BnB,WAAWF,KAAKO,GAAG,CAAC5B,IAAIe,IAAI;oCAClC,IAAIQ,aAAawB,aAAazB,QAAQC,UAAUF,KAAKQ,GAAG,CAAC7B,IAAIe,IAAI,EAAEO;gCACrE;;gCAJK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAKP;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAUL;;wBAAQ,qBAAGD,KAAKY,OAAO,IACpBC,IAAI,CAAC,SAAC7B,GAAGC;mCAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;2BAC1B8B,KAAK,CAAC,GAAGG,KAAKU,CAAC,EACflC,GAAG,CAAC;qEAAEmC,eAAG3B;mCAAY;gCAAEP,MAAMkC;gCAAGb,YAAYC,IAAAA,mBAAQ,EAACf;4BAAO;;;;;IACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, compareVectorScores, sampleEvenly } from '../vectors.ts';\n\n// Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans\n// score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.\n\n// A model narrower than the column zero-pads: the added dimensions contribute nothing to\n// either dot product or norm, so cosine is unchanged.\nfunction padded(values: ArrayLike<number>, dims: number): Float32Array {\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\n// A bound Float32Array stores truncated bytes with no error; a Buffer view over the same\n// bytes round-trips exact (measured 0.7.2).\nfunction toBlob(v: Float32Array): Buffer {\n  return Buffer.from(v.buffer, v.byteOffset, v.byteLength);\n}\n\nfunction decode(vector: Buffer, dims: number): Float32Array {\n  return new Float32Array(vector.buffer, vector.byteOffset, dims);\n}\n\n// Rows arrive int8-quantized with a per-vector scale (embed/query.ts's toStore); this store\n// dequantizes to floats, as duckdb does.\nfunction dequantize(row: VectorWriteRow, dims: number): Float32Array {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(q.length, dims); d++) out[d] = q[d] * row.scale;\n  return out;\n}\n\nfunction cosineSimilarity(a: Float32Array, b: Float32Array): number {\n  let dot = 0;\n  let na = 0;\n  let nb = 0;\n  for (let d = 0; d < a.length; d++) {\n    dot += a[d] * b[d];\n    na += a[d] * a[d];\n    nb += b[d] * b[d];\n  }\n  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-32);\n}\n\n// `scale` stays NULL: the column exists so the row shape matches the other stores, and\n// cosine is scale-invariant once the value is dequantized here.\nexport async function writeVectorBatch(conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [toBlob(dequantize(row, dims)), row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine. GROUP BY with MIN() cannot do this on 0.7.2: the aggregate\n// is right but the bare start_line/end_line come from an arbitrary row in the group.\nexport async function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const q = padded(qv, dims);\n  const stmt = await conn.prepare('SELECT \"path\", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY \"path\", chunk');\n  const rows = (await stmt.all()) as Array<{ path: string; chunk: number; start_line: number; end_line: number; vector: Buffer }>;\n\n  const best = new Map<string, { score: number; chunk: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const score = cosineSimilarity(q, decode(row.vector, dims));\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score || (score === existing.score && row.chunk < existing.chunk)) {\n      best.set(row.path, { score, chunk: row.chunk, lines: `L${row.start_line}-${row.end_line}` });\n    }\n  }\n  return [...best.entries()]\n    .map(([path, value]) => ({ path, ...value }))\n    .sort(compareVectorScores)\n    .slice(0, fetch)\n    .map(({ path, lines, score }) => ({ path, lines, similarity: asCosine(score) }));\n}\n\n// Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured\n// 6-8x slower here at 26k chunks (2026-08-30).\nexport async function scanSimilar(conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Buffer }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => decode(row.vector, dims));\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Buffer }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = decode(row.vector, dims);\n    for (const t of targets) {\n      const score = cosineSimilarity(t, other);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .map(([p, score]) => ({ path: p, score }))\n    .sort(compareVectorScores)\n    .slice(0, opts.k)\n    .map(({ path: p, score }) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","padded","values","dims","out","Float32Array","d","Math","min","length","toBlob","v","Buffer","from","buffer","byteOffset","byteLength","decode","vector","dequantize","row","q","Int8Array","scale","cosineSimilarity","a","b","dot","na","nb","sqrt","conn","rows","runBatch","map","path","chunk","qv","fetch","allowed","stmt","best","score","existing","prepare","all","Map","has","get","set","lines","start_line","end_line","entries","value","sort","compareVectorScores","slice","similarity","asCosine","opts","targetStmt","targetRows","targets","other","t","size","sampleEvenly","exclude","undefined","p","k"],"mappings":";;;;;;;;;;;QAyDsBA;eAAAA;;QAuBAC;eAAAA;;QAjCAC;eAAAA;;;yBA9CsC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5D,0FAA0F;AAC1F,wFAAwF;AAExF,yFAAyF;AACzF,sDAAsD;AACtD,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,IAAMC,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOF;AACT;AAEA,yFAAyF;AACzF,4CAA4C;AAC5C,SAASM,OAAOC,CAAe;IAC7B,OAAOC,OAAOC,IAAI,CAACF,EAAEG,MAAM,EAAEH,EAAEI,UAAU,EAAEJ,EAAEK,UAAU;AACzD;AAEA,SAASC,OAAOC,MAAc,EAAEf,IAAY;IAC1C,OAAO,IAAIE,aAAaa,OAAOJ,MAAM,EAAEI,OAAOH,UAAU,EAAEZ;AAC5D;AAEA,4FAA4F;AAC5F,yCAAyC;AACzC,SAASgB,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,IAAMkB,IAAI,IAAIC,UAAUF,IAAIF,MAAM,CAACJ,MAAM,EAAEM,IAAIF,MAAM,CAACH,UAAU,EAAEK,IAAIF,MAAM,CAACF,UAAU;IACvF,IAAMZ,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACa,EAAEZ,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGe,CAAC,CAACf,EAAE,GAAGc,IAAIG,KAAK;IAC5E,OAAOnB;AACT;AAEA,SAASoB,iBAAiBC,CAAe,EAAEC,CAAe;IACxD,IAAIC,MAAM;IACV,IAAIC,KAAK;IACT,IAAIC,KAAK;IACT,IAAK,IAAIvB,IAAI,GAAGA,IAAImB,EAAEhB,MAAM,EAAEH,IAAK;QACjCqB,OAAOF,CAAC,CAACnB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;QAClBsB,MAAMH,CAAC,CAACnB,EAAE,GAAGmB,CAAC,CAACnB,EAAE;QACjBuB,MAAMH,CAAC,CAACpB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;IACnB;IACA,OAAOqB,MAAOpB,CAAAA,KAAKuB,IAAI,CAACF,MAAMrB,KAAKuB,IAAI,CAACD,MAAM,KAAI;AACpD;AAIO,SAAe7B,iBAAiB+B,IAAgB,EAAE5B,IAAY,EAAE6B,IAAsB;;;;;oBAC3F,IAAIA,KAAKvB,MAAM,KAAK,GAAG;;;oBACvB;;wBAAMsB,KAAKE,QAAQ,CACjB,mEACAD,KAAKE,GAAG,CAAC,SAACd;mCAAQ;gCAACV,OAAOS,WAAWC,KAAKjB;gCAAQiB,IAAIe,IAAI;gCAAEf,IAAIgB,KAAK;6BAAC;;;;oBAFxE;;;;;;IAIF;;AAIO,SAAetC,eAAeiC,IAAgB,EAAEM,EAAgB,EAAElC,IAAY,EAAEmC,KAAa,EAAEC,OAAqB;;YACnHlB,GACAmB,MACAR,MAEAS,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB,KAEHsB,OACAC;;;;oBARFtB,IAAIpB,OAAOoC,IAAIlC;oBACR;;wBAAM4B,KAAKa,OAAO,CAAC;;;oBAA1BJ,OAAO;oBACC;;wBAAMA,KAAKK,GAAG;;;oBAAtBb,OAAQ;oBAERS,OAAO,IAAIK;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAad,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbZ,MAAN;4BACH,IAAImB,WAAW,CAACA,QAAQQ,GAAG,CAAC3B,IAAIe,IAAI,GAAG;4BACjCO,QAAQlB,iBAAiBH,GAAGJ,OAAOG,IAAIF,MAAM,EAAEf;4BAC/CwC,WAAWF,KAAKO,GAAG,CAAC5B,IAAIe,IAAI;4BAClC,IAAI,CAACQ,YAAYD,QAAQC,SAASD,KAAK,IAAKA,UAAUC,SAASD,KAAK,IAAItB,IAAIgB,KAAK,GAAGO,SAASP,KAAK,EAAG;gCACnGK,KAAKQ,GAAG,CAAC7B,IAAIe,IAAI,EAAE;oCAAEO,OAAAA;oCAAON,OAAOhB,IAAIgB,KAAK;oCAAEc,OAAO,AAAC,IAAqB9B,OAAlBA,IAAI+B,UAAU,EAAC,KAAgB,OAAb/B,IAAIgC,QAAQ;gCAAG;4BAC5F;wBACF;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAQL;;wBAAQ,qBAAGX,KAAKY,OAAO,IACpBnB,GAAG,CAAC;qEAAEC,kBAAMmB;mCAAY;gCAAEnB,MAAAA;+BAASmB;2BACnCC,IAAI,CAACC,8BAAmB,EACxBC,KAAK,CAAC,GAAGnB,OACTJ,GAAG,CAAC;gCAAGC,aAAAA,MAAMe,cAAAA,OAAOR,cAAAA;mCAAa;gCAAEP,MAAAA;gCAAMe,OAAAA;gCAAOQ,YAAYC,IAAAA,mBAAQ,EAACjB;4BAAO;;;;;IACjF;;AAIO,SAAe3C,YAAYgC,IAAgB,EAAE5B,IAAY,EAAEgC,IAAY,EAAEyB,IAAgE;;YAExIC,YACAC,YAEAC,SAEAvB,MACAR,MACAS,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB,KAEH4C,OACD,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,GACHvB,OACAC;;;;oBAdV,IAAIiB,KAAKrB,OAAO,IAAIqB,KAAKrB,OAAO,CAAC2B,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAMnC,KAAKa,OAAO,CAAC;;;oBAAhCiB,aAAa;oBACC;;wBAAMA,WAAWhB,GAAG,CAACV;;;oBAAnC2B,aAAc;oBACpB,IAAIA,WAAWrD,MAAM,KAAK,GAAG;;;;oBACvBsD,UAAUI,IAAAA,uBAAY,EAACL,YAAY5B,GAAG,CAAC,SAACd;+BAAQH,OAAOG,IAAIF,MAAM,EAAEf;;oBAE5D;;wBAAM4B,KAAKa,OAAO,CAAC;;;oBAA1BJ,OAAO;oBACC;;wBAAMA,KAAKK,GAAG;;;oBAAtBb,OAAQ;oBACRS,OAAO,IAAIK;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAad,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbZ,MAAN;4BACH,IAAIA,IAAIe,IAAI,KAAKA,QAAQyB,KAAKQ,OAAO,CAACrB,GAAG,CAAC3B,IAAIe,IAAI,KAAMyB,KAAKrB,OAAO,IAAI,CAACqB,KAAKrB,OAAO,CAACQ,GAAG,CAAC3B,IAAIe,IAAI,GAAI;4BAChG6B,QAAQ/C,OAAOG,IAAIF,MAAM,EAAEf;4BAC5B,mCAAA,4BAAA;;gCAAL,IAAK,aAAW4D,8BAAX,8BAAA,SAAA,0BAAA,kCAAoB;oCAAdE,IAAN;oCACGvB,QAAQlB,iBAAiByC,GAAGD;oCAC5BrB,WAAWF,KAAKO,GAAG,CAAC5B,IAAIe,IAAI;oCAClC,IAAIQ,aAAa0B,aAAa3B,QAAQC,UAAUF,KAAKQ,GAAG,CAAC7B,IAAIe,IAAI,EAAEO;gCACrE;;gCAJK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAKP;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAUL;;wBAAQ,qBAAGD,KAAKY,OAAO,IACpBnB,GAAG,CAAC;qEAAEoC,eAAG5B;mCAAY;gCAAEP,MAAMmC;gCAAG5B,OAAAA;4BAAM;2BACtCa,IAAI,CAACC,8BAAmB,EACxBC,KAAK,CAAC,GAAGG,KAAKW,CAAC,EACfrC,GAAG,CAAC;gCAAGC,AAAMmC,UAANnC,MAASO,cAAAA;mCAAa;gCAAEP,MAAMmC;gCAAGZ,YAAYC,IAAAA,mBAAQ,EAACjB;4BAAO;;;;;IACzE"}

package/dist/cjs/store/types.d.cts

@@ -1,7 +1,7 @@
 import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
-export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "lexical", "vectors", "sql-functions"];
+export declare const CAPABILITY_NAMES: readonly ["vectors", "segment"];
 export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;
@@ -55,7 +55,6 @@ export interface DocumentStore {
 }
 export interface LexicalHit {
     path: string;
-    hit: string | null;
 }
 export interface LexicalQueryOptions {
     whereJoin: string;

package/dist/cjs/store/types.d.ts

@@ -1,7 +1,7 @@
 import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
-export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "lexical", "vectors", "sql-functions"];
+export declare const CAPABILITY_NAMES: readonly ["vectors", "segment"];
 export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;
@@ -55,7 +55,6 @@ export interface DocumentStore {
 }
 export interface LexicalHit {
     path: string;
-    hit: string | null;
 }
 export interface LexicalQueryOptions {
     whereJoin: string;

package/dist/cjs/store/types.js

@@ -11,10 +11,7 @@ Object.defineProperty(exports, "CAPABILITY_NAMES", {
     }
 });
 var CAPABILITY_NAMES = [
-    'phrases',
-    'snippets',
-    'lexical',
     'vectors',
-    'sql-functions'
+    'segment'
 ];
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }