sensemaking 0.23.0 → 0.24.0

package/dist/esm/store/turso/sql-functions.js

@@ -0,0 +1,179 @@
+import { SenseError } from '../../errors.js';
+// Turso can't register UDFs: has()/basename() are rewritten into portable SQL below; segment()
+// (Intl.Segmenter grapheme clustering, types.ts's 'segment' capability) has none, so it's rejected like an unsupported FTS5 operator.
+// Non-global: recursion (readArgs' contents rewritten via rewriteFunctions) reenters this module's
+// regexes before the outer scan resumes, and a global regex's lastIndex does not nest across that.
+const CALL_ANYWHERE = /\b(has|basename|segment)\s*\(/i;
+const CALL_AT_START = /^(has|basename|segment)\s*\(/i;
+function isWordChar(ch) {
+    return ch !== undefined && /[A-Za-z0-9_]/.test(ch);
+}
+// The index just past the quoted literal or SQL comment starting at i, or null if i starts
+// neither. Shared by the scan and readArgs so both leave quoted/comment text opaque to a call match.
+function skipOpaque(sql, i) {
+    const ch = sql[i];
+    if (ch === "'" || ch === '"') {
+        let j = i + 1;
+        while(j < sql.length){
+            if (sql[j] === ch) {
+                if (sql[j + 1] === ch) {
+                    j += 2;
+                    continue;
+                }
+                return j + 1;
+            }
+            j++;
+        }
+        return j; // unterminated literal: consume to end rather than loop forever
+    }
+    if (ch === '-' && sql[i + 1] === '-') {
+        const nl = sql.indexOf('\n', i);
+        return nl === -1 ? sql.length : nl + 1;
+    }
+    if (ch === '/' && sql[i + 1] === '*') {
+        const close = sql.indexOf('*/', i + 2);
+        return close === -1 ? sql.length : close + 2;
+    }
+    return null;
+}
+// `?N`/named placeholders bind by number/name, so repeating one is safe; an anonymous `?` takes
+// the next slot per occurrence, so it's numbered here first -- sqlHas/sqlBasename duplicate their arguments.
+function numberPlaceholders(sql) {
+    let out = '';
+    let i = 0;
+    let n = 0;
+    while(i < sql.length){
+        var _sql_;
+        const skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            out += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        if (sql[i] === '?' && !/[0-9]/.test((_sql_ = sql[i + 1]) !== null && _sql_ !== void 0 ? _sql_ : '')) {
+            n++;
+            out += `?${n}`;
+            i++;
+            continue;
+        }
+        out += sql[i];
+        i++;
+    }
+    return out;
+}
+// The un-rewritten argument texts of the call whose '(' sits at parenStart, and the index just
+// past its matching ')'. Paren depth is tracked so a nested call's commas/parens aren't mistaken for ours.
+function readArgs(sql, parenStart) {
+    const args = [];
+    let depth = 1;
+    let cur = '';
+    let i = parenStart + 1;
+    while(i < sql.length){
+        const skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            cur += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        const ch = sql[i];
+        if (ch === '(') {
+            depth++;
+            cur += ch;
+            i++;
+            continue;
+        }
+        if (ch === ')') {
+            depth--;
+            i++;
+            if (depth === 0) break;
+            cur += ch;
+            continue;
+        }
+        if (ch === ',' && depth === 1) {
+            args.push(cur.trim());
+            cur = '';
+            i++;
+            continue;
+        }
+        cur += ch;
+        i++;
+    }
+    if (depth !== 0) throw new Error(`turso: unterminated argument list starting at "${sql.slice(parenStart, parenStart + 30)}..."`);
+    if (cur.trim() !== '') args.push(cur.trim());
+    return {
+        args,
+        end: i
+    };
+}
+// field.startsWith('[') && JSON.parse(field) is an array, matched by membership; every other
+// field, stringified, matched by substring (sql-functions.ts's hasImpl, mirrored branch for branch).
+function sqlHas(fieldExpr, valueExpr) {
+    const isArray = `substr(${fieldExpr},1,1) = '[' AND json_valid(${fieldExpr})`;
+    // json_each's argument is re-guarded by its own CASE (falling back to '[]'): turso evaluates a
+    // CASE branch's table function even when untaken, and a non-array argument there raises rather than yielding 0 rows (spike-verified).
+    return `(CASE
+    WHEN ${fieldExpr} IS NULL THEN 0
+    WHEN ${isArray} THEN
+      (SELECT COUNT(*) FROM json_each(CASE WHEN ${isArray} THEN ${fieldExpr} ELSE '[]' END) je WHERE CAST(je.value AS TEXT) = CAST(${valueExpr} AS TEXT)) > 0
+    ELSE instr(CAST(${fieldExpr} AS TEXT), CAST(${valueExpr} AS TEXT)) > 0
+  END)`;
+}
+// posix.basename(String(path)), suffix stripped Unix-style (sql-functions.ts's basenameImpl). No
+// last-'/' primitive exists, so reverse() finds the first '/' from the end, then reverse() restores order.
+function sqlBasename(pathExpr, suffixExpr) {
+    const trimmed = `rtrim(CAST(${pathExpr} AS TEXT), '/')`;
+    const name = `CASE
+      WHEN ${trimmed} = '' THEN ''
+      WHEN instr(${trimmed}, '/') = 0 THEN ${trimmed}
+      ELSE reverse(substr(reverse(${trimmed}), 1, instr(reverse(${trimmed}), '/') - 1))
+    END`;
+    return `(CASE WHEN ${pathExpr} IS NULL THEN NULL ELSE
+    (CASE WHEN ${suffixExpr} IS NOT NULL AND CAST(${suffixExpr} AS TEXT) != '' AND CAST(${suffixExpr} AS TEXT) != (${name})
+          AND substr((${name}), -length(CAST(${suffixExpr} AS TEXT))) = CAST(${suffixExpr} AS TEXT)
+     THEN substr((${name}), 1, length((${name})) - length(CAST(${suffixExpr} AS TEXT)))
+     ELSE (${name}) END)
+  END)`;
+}
+// Rewrites every has()/basename() call into the portable SQL above, recursing into each call's own
+// arguments first so a nested `instr(basename(f.path), 'x')` resolves inside out; segment() instead raises STORE_CAPABILITY_MISSING, named, like an unsupported FTS5 operator.
+export function rewriteFunctions(sql) {
+    if (!CALL_ANYWHERE.test(sql)) return sql;
+    // Numbered once, over the whole statement, before any argument is duplicated below -- rewrite()
+    // recurses over already-numbered substrings, never renumbering them a second time.
+    return rewrite(numberPlaceholders(sql));
+}
+function rewrite(sql) {
+    let out = '';
+    let i = 0;
+    while(i < sql.length){
+        const skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            out += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        const m = !isWordChar(sql[i - 1]) ? CALL_AT_START.exec(sql.slice(i)) : null;
+        if (m) {
+            const name = m[1].toLowerCase();
+            const parenStart = i + m[0].length - 1;
+            if (name === 'segment') {
+                throw new SenseError('STORE_CAPABILITY_MISSING', `store "turso" does not implement segment() as a SQL function in this build (its client cannot register UDFs); rephrase the query without it, or set "store" to "sqlite" in this tree's config to run it as written`);
+            }
+            const { args, end } = readArgs(sql, parenStart);
+            const rewrittenArgs = args.map(rewrite);
+            if (name === 'has') {
+                if (rewrittenArgs.length !== 2) throw new Error(`turso: has() takes exactly 2 arguments (field, value), got ${rewrittenArgs.length}`);
+                out += sqlHas(rewrittenArgs[0], rewrittenArgs[1]);
+            } else {
+                var _rewrittenArgs_;
+                if (rewrittenArgs.length < 1 || rewrittenArgs.length > 2) throw new Error(`turso: basename() takes 1 or 2 arguments (path, suffix?), got ${rewrittenArgs.length}`);
+                out += sqlBasename(rewrittenArgs[0], (_rewrittenArgs_ = rewrittenArgs[1]) !== null && _rewrittenArgs_ !== void 0 ? _rewrittenArgs_ : 'NULL');
+            }
+            i = end;
+            continue;
+        }
+        out += sql[i];
+        i++;
+    }
+    return out;
+}

package/dist/esm/store/turso/sql-functions.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/sql-functions.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\n\n// Turso can't register UDFs: has()/basename() are rewritten into portable SQL below; segment()\n// (Intl.Segmenter grapheme clustering, types.ts's 'segment' capability) has none, so it's rejected like an unsupported FTS5 operator.\n\n// Non-global: recursion (readArgs' contents rewritten via rewriteFunctions) reenters this module's\n// regexes before the outer scan resumes, and a global regex's lastIndex does not nest across that.\nconst CALL_ANYWHERE = /\\b(has|basename|segment)\\s*\\(/i;\nconst CALL_AT_START = /^(has|basename|segment)\\s*\\(/i;\n\nfunction isWordChar(ch: string | undefined): boolean {\n  return ch !== undefined && /[A-Za-z0-9_]/.test(ch);\n}\n\n// The index just past the quoted literal or SQL comment starting at i, or null if i starts\n// neither. Shared by the scan and readArgs so both leave quoted/comment text opaque to a call match.\nfunction skipOpaque(sql: string, i: number): number | null {\n  const ch = sql[i];\n  if (ch === \"'\" || ch === '\"') {\n    let j = i + 1;\n    while (j < sql.length) {\n      if (sql[j] === ch) {\n        if (sql[j + 1] === ch) {\n          j += 2;\n          continue;\n        }\n        return j + 1;\n      }\n      j++;\n    }\n    return j; // unterminated literal: consume to end rather than loop forever\n  }\n  if (ch === '-' && sql[i + 1] === '-') {\n    const nl = sql.indexOf('\\n', i);\n    return nl === -1 ? sql.length : nl + 1;\n  }\n  if (ch === '/' && sql[i + 1] === '*') {\n    const close = sql.indexOf('*/', i + 2);\n    return close === -1 ? sql.length : close + 2;\n  }\n  return null;\n}\n\n// `?N`/named placeholders bind by number/name, so repeating one is safe; an anonymous `?` takes\n// the next slot per occurrence, so it's numbered here first -- sqlHas/sqlBasename duplicate their arguments.\nfunction numberPlaceholders(sql: string): string {\n  let out = '';\n  let i = 0;\n  let n = 0;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      out += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    if (sql[i] === '?' && !/[0-9]/.test(sql[i + 1] ?? '')) {\n      n++;\n      out += `?${n}`;\n      i++;\n      continue;\n    }\n    out += sql[i];\n    i++;\n  }\n  return out;\n}\n\n// The un-rewritten argument texts of the call whose '(' sits at parenStart, and the index just\n// past its matching ')'. Paren depth is tracked so a nested call's commas/parens aren't mistaken for ours.\nfunction readArgs(sql: string, parenStart: number): { args: string[]; end: number } {\n  const args: string[] = [];\n  let depth = 1;\n  let cur = '';\n  let i = parenStart + 1;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      cur += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    const ch = sql[i];\n    if (ch === '(') {\n      depth++;\n      cur += ch;\n      i++;\n      continue;\n    }\n    if (ch === ')') {\n      depth--;\n      i++;\n      if (depth === 0) break;\n      cur += ch;\n      continue;\n    }\n    if (ch === ',' && depth === 1) {\n      args.push(cur.trim());\n      cur = '';\n      i++;\n      continue;\n    }\n    cur += ch;\n    i++;\n  }\n  if (depth !== 0) throw new Error(`turso: unterminated argument list starting at \"${sql.slice(parenStart, parenStart + 30)}...\"`);\n  if (cur.trim() !== '') args.push(cur.trim());\n  return { args, end: i };\n}\n\n// field.startsWith('[') && JSON.parse(field) is an array, matched by membership; every other\n// field, stringified, matched by substring (sql-functions.ts's hasImpl, mirrored branch for branch).\nfunction sqlHas(fieldExpr: string, valueExpr: string): string {\n  const isArray = `substr(${fieldExpr},1,1) = '[' AND json_valid(${fieldExpr})`;\n  // json_each's argument is re-guarded by its own CASE (falling back to '[]'): turso evaluates a\n  // CASE branch's table function even when untaken, and a non-array argument there raises rather than yielding 0 rows (spike-verified).\n  return `(CASE\n    WHEN ${fieldExpr} IS NULL THEN 0\n    WHEN ${isArray} THEN\n      (SELECT COUNT(*) FROM json_each(CASE WHEN ${isArray} THEN ${fieldExpr} ELSE '[]' END) je WHERE CAST(je.value AS TEXT) = CAST(${valueExpr} AS TEXT)) > 0\n    ELSE instr(CAST(${fieldExpr} AS TEXT), CAST(${valueExpr} AS TEXT)) > 0\n  END)`;\n}\n\n// posix.basename(String(path)), suffix stripped Unix-style (sql-functions.ts's basenameImpl). No\n// last-'/' primitive exists, so reverse() finds the first '/' from the end, then reverse() restores order.\nfunction sqlBasename(pathExpr: string, suffixExpr: string): string {\n  const trimmed = `rtrim(CAST(${pathExpr} AS TEXT), '/')`;\n  const name = `CASE\n      WHEN ${trimmed} = '' THEN ''\n      WHEN instr(${trimmed}, '/') = 0 THEN ${trimmed}\n      ELSE reverse(substr(reverse(${trimmed}), 1, instr(reverse(${trimmed}), '/') - 1))\n    END`;\n  return `(CASE WHEN ${pathExpr} IS NULL THEN NULL ELSE\n    (CASE WHEN ${suffixExpr} IS NOT NULL AND CAST(${suffixExpr} AS TEXT) != '' AND CAST(${suffixExpr} AS TEXT) != (${name})\n          AND substr((${name}), -length(CAST(${suffixExpr} AS TEXT))) = CAST(${suffixExpr} AS TEXT)\n     THEN substr((${name}), 1, length((${name})) - length(CAST(${suffixExpr} AS TEXT)))\n     ELSE (${name}) END)\n  END)`;\n}\n\n// Rewrites every has()/basename() call into the portable SQL above, recursing into each call's own\n// arguments first so a nested `instr(basename(f.path), 'x')` resolves inside out; segment() instead raises STORE_CAPABILITY_MISSING, named, like an unsupported FTS5 operator.\nexport function rewriteFunctions(sql: string): string {\n  if (!CALL_ANYWHERE.test(sql)) return sql;\n  // Numbered once, over the whole statement, before any argument is duplicated below -- rewrite()\n  // recurses over already-numbered substrings, never renumbering them a second time.\n  return rewrite(numberPlaceholders(sql));\n}\n\nfunction rewrite(sql: string): string {\n  let out = '';\n  let i = 0;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      out += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    const m = !isWordChar(sql[i - 1]) ? CALL_AT_START.exec(sql.slice(i)) : null;\n    if (m) {\n      const name = m[1].toLowerCase();\n      const parenStart = i + m[0].length - 1;\n      if (name === 'segment') {\n        throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement segment() as a SQL function in this build (its client cannot register UDFs); rephrase the query without it, or set \"store\" to \"sqlite\" in this tree's config to run it as written`);\n      }\n      const { args, end } = readArgs(sql, parenStart);\n      const rewrittenArgs = args.map(rewrite);\n      if (name === 'has') {\n        if (rewrittenArgs.length !== 2) throw new Error(`turso: has() takes exactly 2 arguments (field, value), got ${rewrittenArgs.length}`);\n        out += sqlHas(rewrittenArgs[0], rewrittenArgs[1]);\n      } else {\n        if (rewrittenArgs.length < 1 || rewrittenArgs.length > 2) throw new Error(`turso: basename() takes 1 or 2 arguments (path, suffix?), got ${rewrittenArgs.length}`);\n        out += sqlBasename(rewrittenArgs[0], rewrittenArgs[1] ?? 'NULL');\n      }\n      i = end;\n      continue;\n    }\n    out += sql[i];\n    i++;\n  }\n  return out;\n}\n"],"names":["SenseError","CALL_ANYWHERE","CALL_AT_START","isWordChar","ch","undefined","test","skipOpaque","sql","i","j","length","nl","indexOf","close","numberPlaceholders","out","n","skip","slice","readArgs","parenStart","args","depth","cur","push","trim","Error","end","sqlHas","fieldExpr","valueExpr","isArray","sqlBasename","pathExpr","suffixExpr","trimmed","name","rewriteFunctions","rewrite","m","exec","toLowerCase","rewrittenArgs","map"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAE7C,+FAA+F;AAC/F,sIAAsI;AAEtI,mGAAmG;AACnG,mGAAmG;AACnG,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,SAASC,WAAWC,EAAsB;IACxC,OAAOA,OAAOC,aAAa,eAAeC,IAAI,CAACF;AACjD;AAEA,2FAA2F;AAC3F,qGAAqG;AACrG,SAASG,WAAWC,GAAW,EAAEC,CAAS;IACxC,MAAML,KAAKI,GAAG,CAACC,EAAE;IACjB,IAAIL,OAAO,OAAOA,OAAO,KAAK;QAC5B,IAAIM,IAAID,IAAI;QACZ,MAAOC,IAAIF,IAAIG,MAAM,CAAE;YACrB,IAAIH,GAAG,CAACE,EAAE,KAAKN,IAAI;gBACjB,IAAII,GAAG,CAACE,IAAI,EAAE,KAAKN,IAAI;oBACrBM,KAAK;oBACL;gBACF;gBACA,OAAOA,IAAI;YACb;YACAA;QACF;QACA,OAAOA,GAAG,gEAAgE;IAC5E;IACA,IAAIN,OAAO,OAAOI,GAAG,CAACC,IAAI,EAAE,KAAK,KAAK;QACpC,MAAMG,KAAKJ,IAAIK,OAAO,CAAC,MAAMJ;QAC7B,OAAOG,OAAO,CAAC,IAAIJ,IAAIG,MAAM,GAAGC,KAAK;IACvC;IACA,IAAIR,OAAO,OAAOI,GAAG,CAACC,IAAI,EAAE,KAAK,KAAK;QACpC,MAAMK,QAAQN,IAAIK,OAAO,CAAC,MAAMJ,IAAI;QACpC,OAAOK,UAAU,CAAC,IAAIN,IAAIG,MAAM,GAAGG,QAAQ;IAC7C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,6GAA6G;AAC7G,SAASC,mBAAmBP,GAAW;IACrC,IAAIQ,MAAM;IACV,IAAIP,IAAI;IACR,IAAIQ,IAAI;IACR,MAAOR,IAAID,IAAIG,MAAM,CAAE;YAOeH;QANpC,MAAMU,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBF,OAAOR,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,IAAIV,GAAG,CAACC,EAAE,KAAK,OAAO,CAAC,QAAQH,IAAI,EAACE,QAAAA,GAAG,CAACC,IAAI,EAAE,cAAVD,mBAAAA,QAAc,KAAK;YACrDS;YACAD,OAAO,CAAC,CAAC,EAAEC,GAAG;YACdR;YACA;QACF;QACAO,OAAOR,GAAG,CAACC,EAAE;QACbA;IACF;IACA,OAAOO;AACT;AAEA,+FAA+F;AAC/F,2GAA2G;AAC3G,SAASI,SAASZ,GAAW,EAAEa,UAAkB;IAC/C,MAAMC,OAAiB,EAAE;IACzB,IAAIC,QAAQ;IACZ,IAAIC,MAAM;IACV,IAAIf,IAAIY,aAAa;IACrB,MAAOZ,IAAID,IAAIG,MAAM,CAAE;QACrB,MAAMO,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBM,OAAOhB,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,MAAMd,KAAKI,GAAG,CAACC,EAAE;QACjB,IAAIL,OAAO,KAAK;YACdmB;YACAC,OAAOpB;YACPK;YACA;QACF;QACA,IAAIL,OAAO,KAAK;YACdmB;YACAd;YACA,IAAIc,UAAU,GAAG;YACjBC,OAAOpB;YACP;QACF;QACA,IAAIA,OAAO,OAAOmB,UAAU,GAAG;YAC7BD,KAAKG,IAAI,CAACD,IAAIE,IAAI;YAClBF,MAAM;YACNf;YACA;QACF;QACAe,OAAOpB;QACPK;IACF;IACA,IAAIc,UAAU,GAAG,MAAM,IAAII,MAAM,CAAC,+CAA+C,EAAEnB,IAAIW,KAAK,CAACE,YAAYA,aAAa,IAAI,IAAI,CAAC;IAC/H,IAAIG,IAAIE,IAAI,OAAO,IAAIJ,KAAKG,IAAI,CAACD,IAAIE,IAAI;IACzC,OAAO;QAAEJ;QAAMM,KAAKnB;IAAE;AACxB;AAEA,6FAA6F;AAC7F,qGAAqG;AACrG,SAASoB,OAAOC,SAAiB,EAAEC,SAAiB;IAClD,MAAMC,UAAU,CAAC,OAAO,EAAEF,UAAU,2BAA2B,EAAEA,UAAU,CAAC,CAAC;IAC7E,+FAA+F;IAC/F,sIAAsI;IACtI,OAAO,CAAC;SACD,EAAEA,UAAU;SACZ,EAAEE,QAAQ;gDAC6B,EAAEA,QAAQ,MAAM,EAAEF,UAAU,uDAAuD,EAAEC,UAAU;oBAC3H,EAAED,UAAU,gBAAgB,EAAEC,UAAU;MACtD,CAAC;AACP;AAEA,iGAAiG;AACjG,2GAA2G;AAC3G,SAASE,YAAYC,QAAgB,EAAEC,UAAkB;IACvD,MAAMC,UAAU,CAAC,WAAW,EAAEF,SAAS,eAAe,CAAC;IACvD,MAAMG,OAAO,CAAC;WACL,EAAED,QAAQ;iBACJ,EAAEA,QAAQ,gBAAgB,EAAEA,QAAQ;kCACnB,EAAEA,QAAQ,oBAAoB,EAAEA,QAAQ;OACnE,CAAC;IACN,OAAO,CAAC,WAAW,EAAEF,SAAS;eACjB,EAAEC,WAAW,sBAAsB,EAAEA,WAAW,yBAAyB,EAAEA,WAAW,cAAc,EAAEE,KAAK;sBACpG,EAAEA,KAAK,gBAAgB,EAAEF,WAAW,mBAAmB,EAAEA,WAAW;kBACxE,EAAEE,KAAK,cAAc,EAAEA,KAAK,iBAAiB,EAAEF,WAAW;WACjE,EAAEE,KAAK;MACZ,CAAC;AACP;AAEA,mGAAmG;AACnG,+KAA+K;AAC/K,OAAO,SAASC,iBAAiB9B,GAAW;IAC1C,IAAI,CAACP,cAAcK,IAAI,CAACE,MAAM,OAAOA;IACrC,gGAAgG;IAChG,mFAAmF;IACnF,OAAO+B,QAAQxB,mBAAmBP;AACpC;AAEA,SAAS+B,QAAQ/B,GAAW;IAC1B,IAAIQ,MAAM;IACV,IAAIP,IAAI;IACR,MAAOA,IAAID,IAAIG,MAAM,CAAE;QACrB,MAAMO,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBF,OAAOR,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,MAAMsB,IAAI,CAACrC,WAAWK,GAAG,CAACC,IAAI,EAAE,IAAIP,cAAcuC,IAAI,CAACjC,IAAIW,KAAK,CAACV,MAAM;QACvE,IAAI+B,GAAG;YACL,MAAMH,OAAOG,CAAC,CAAC,EAAE,CAACE,WAAW;YAC7B,MAAMrB,aAAaZ,IAAI+B,CAAC,CAAC,EAAE,CAAC7B,MAAM,GAAG;YACrC,IAAI0B,SAAS,WAAW;gBACtB,MAAM,IAAIrC,WAAW,4BAA4B,CAAC,kNAAkN,CAAC;YACvQ;YACA,MAAM,EAAEsB,IAAI,EAAEM,GAAG,EAAE,GAAGR,SAASZ,KAAKa;YACpC,MAAMsB,gBAAgBrB,KAAKsB,GAAG,CAACL;YAC/B,IAAIF,SAAS,OAAO;gBAClB,IAAIM,cAAchC,MAAM,KAAK,GAAG,MAAM,IAAIgB,MAAM,CAAC,2DAA2D,EAAEgB,cAAchC,MAAM,EAAE;gBACpIK,OAAOa,OAAOc,aAAa,CAAC,EAAE,EAAEA,aAAa,CAAC,EAAE;YAClD,OAAO;oBAEgCA;gBADrC,IAAIA,cAAchC,MAAM,GAAG,KAAKgC,cAAchC,MAAM,GAAG,GAAG,MAAM,IAAIgB,MAAM,CAAC,8DAA8D,EAAEgB,cAAchC,MAAM,EAAE;gBACjKK,OAAOiB,YAAYU,aAAa,CAAC,EAAE,GAAEA,kBAAAA,aAAa,CAAC,EAAE,cAAhBA,6BAAAA,kBAAoB;YAC3D;YACAlC,IAAImB;YACJ;QACF;QACAZ,OAAOR,GAAG,CAACC,EAAE;QACbA;IACF;IACA,OAAOO;AACT"}

package/dist/esm/store/turso/store.js

@@ -1,18 +1,19 @@
 import { STORE_DIMS } from '../../embed/types.js';
-import { getColumns } from '../shared.js';
+import { getColumns, getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
-import { checkpointWal } from './connection.js';
+import { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
+import { rewriteFunctions } from './sql-functions.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
-// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use
-// the caller's JS excerpt.
+// No 'segment': has()/basename() resolve via SQL rewrite below (sql-functions.ts), but segment() has no SQL form and the client cannot register it as a UDF.
 export const CAPABILITIES = new Set([
-    'lexical',
-    'phrases',
     'vectors'
 ]);
+// close() reclaims once the cache file outgrows its compact size by this factor: the disk
+// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.
+const BLOAT_FACTOR = 1.5;
 // Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth
 // (see transaction.ts) is tracked against the same object everywhere.
 export function createStore(db, conn) {
@@ -61,7 +62,7 @@ export function createStore(db, conn) {
         },
         raw: {
             async prepare (sql) {
-                const stmt = await db.prepare(sql);
+                const stmt = await db.prepare(rewriteFunctions(sql));
                 stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision
                 return {
                     columns: ()=>stmt.columns(),
@@ -73,8 +74,32 @@ export function createStore(db, conn) {
             }
         },
         async close () {
+            // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.
             await checkpointWal(db);
+            let reclaimPath = null;
+            try {
+                var _ref;
+                const path = await cacheFilePath(conn);
+                const base = Number((_ref = await getMeta(conn, 'compact_size')) !== null && _ref !== void 0 ? _ref : '0');
+                const size = path === null ? null : await fileSize(path);
+                if (size !== null) {
+                    // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a
+                    // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.
+                    if (base === 0) {
+                        await setMeta(conn, 'compact_size', String(size));
+                        // The meta write landed past the checkpoint; leave no WAL for the next opener.
+                        await checkpointWal(db);
+                    } else if (size > BLOAT_FACTOR * base) {
+                        reclaimPath = path;
+                    }
+                }
+            } catch (err) {
+                console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${err.message}`);
+            }
             await db.close();
+            // After this store's connection is closed, never before: turso hands the file to one
+            // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).
+            if (reclaimPath !== null) await reclaimSpace(reclaimPath);
         }
     };
 }

package/dist/esm/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { checkpointWal } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await checkpointWal(db);\n      await db.close();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","checkpointWal","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,aAAa,QAAQ,kBAAkB;AAChD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,2BAA2B;AAC3B,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACd,MAAMO,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOtB,gBAAgBa,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMzB,WAAWc;iBAAO;YACtC;YACAT,YAAY,CAACoB,SAASC,aAAerB,WAAWS,MAAMW,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAASxB,aAAaQ,MAAMe,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAM7B,YAAYW;YAC3BmB,cAAc,CAACC,OAASzB,iBAAiBK,MAAMf,YAAYmC;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,MAAMsB,IAAIrC,YAAYuC,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAAStB,YAAYM,MAAMf,YAAY0C,MAAMX;YAC7DY,WAAW,CAACD,OAASvC,aAAaY,MAAM2B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAM/B,GAAGM,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMnC,GAAGM,OAAO,CAACD;gBAC9B8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMhD,cAAcS;YACpB,MAAMA,GAAGuC,KAAK;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns, getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'segment': has()/basename() resolve via SQL rewrite below (sql-functions.ts), but segment() has no SQL form and the client cannot register it as a UDF.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors']);\n\n// close() reclaims once the cache file outgrows its compact size by this factor: the disk\n// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.\nconst BLOAT_FACTOR = 1.5;\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(rewriteFunctions(sql));\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.\n      await checkpointWal(db);\n      let reclaimPath: string | null = null;\n      try {\n        const path = await cacheFilePath(conn);\n        const base = Number((await getMeta(conn, 'compact_size')) ?? '0');\n        const size = path === null ? null : await fileSize(path);\n        if (size !== null) {\n          // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a\n          // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.\n          if (base === 0) {\n            await setMeta(conn, 'compact_size', String(size));\n            // The meta write landed past the checkpoint; leave no WAL for the next opener.\n            await checkpointWal(db);\n          } else if (size > BLOAT_FACTOR * base) {\n            reclaimPath = path;\n          }\n        }\n      } catch (err) {\n        console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${(err as Error).message}`);\n      }\n      await db.close();\n      // After this store's connection is closed, never before: turso hands the file to one\n      // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).\n      if (reclaimPath !== null) await reclaimSpace(reclaimPath);\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","getMeta","setMeta","withTransaction","hasVectorRow","pendingRows","cacheFilePath","checkpointWal","fileSize","reclaimSpace","fieldStats","queryLexical","rewriteFunctions","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","BLOAT_FACTOR","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close","reclaimPath","base","Number","size","String","err","console","error","message"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,EAAEC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAC5D,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,aAAa,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,YAAY,QAAQ,kBAAkB;AACvF,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,gBAAgB,QAAQ,qBAAqB;AACtD,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,6JAA6J;AAC7J,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;CAAU,EAAE;AAE1E,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,eAAe;AAErB,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcP;QACd,MAAMQ,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBkB,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWqB;iBAAO;YACtC;YACAX,YAAY,CAACsB,SAASC,aAAevB,WAAWW,MAAMW,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAAS1B,aAAaU,MAAMe,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAMlC,YAAYgB;YAC3BmB,cAAc,CAACC,OAAS1B,iBAAiBM,MAAMtB,YAAY0C;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYjC,eAAeQ,MAAMsB,IAAI5C,YAAY8C,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAASvB,YAAYO,MAAMtB,YAAYiD,MAAMX;YAC7DY,WAAW,CAACD,OAAS5C,aAAaiB,MAAM2B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAM/B,GAAGM,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMnC,GAAGM,OAAO,CAACd,iBAAiBa;gBAC/C8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,4FAA4F;YAC5F,MAAMpD,cAAca;YACpB,IAAIwC,cAA6B;YACjC,IAAI;oBAEmB;gBADrB,MAAMZ,OAAO,MAAM1C,cAAce;gBACjC,MAAMwC,OAAOC,QAAQ,OAAA,MAAM7D,QAAQoB,MAAM,6BAApB,kBAAA,OAAwC;gBAC7D,MAAM0C,OAAOf,SAAS,OAAO,OAAO,MAAMxC,SAASwC;gBACnD,IAAIe,SAAS,MAAM;oBACjB,yFAAyF;oBACzF,2FAA2F;oBAC3F,IAAIF,SAAS,GAAG;wBACd,MAAM3D,QAAQmB,MAAM,gBAAgB2C,OAAOD;wBAC3C,+EAA+E;wBAC/E,MAAMxD,cAAca;oBACtB,OAAO,IAAI2C,OAAO7C,eAAe2C,MAAM;wBACrCD,cAAcZ;oBAChB;gBACF;YACF,EAAE,OAAOiB,KAAK;gBACZC,QAAQC,KAAK,CAAC,CAAC,yFAAyF,EAAE,AAACF,IAAcG,OAAO,EAAE;YACpI;YACA,MAAMhD,GAAGuC,KAAK;YACd,qFAAqF;YACrF,+FAA+F;YAC/F,IAAIC,gBAAgB,MAAM,MAAMnD,aAAamD;QAC/C;IACF;AACF"}

package/dist/esm/store/turso/vectors.js

@@ -1,4 +1,4 @@
-import { asCosine, sampleEvenly } from '../vectors.js';
+import { asCosine, compareVectorScores, sampleEvenly } from '../vectors.js';
 // Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans
 // score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.
 // A model narrower than the column zero-pads: the added dimensions contribute nothing to
@@ -49,24 +49,30 @@ export async function writeVectorBatch(conn, dims, rows) {
 // is right but the bare start_line/end_line come from an arbitrary row in the group.
 export async function scanCandidates(conn, qv, dims, fetch, allowed) {
     const q = padded(qv, dims);
-    const stmt = await conn.prepare('SELECT "path", start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL');
+    const stmt = await conn.prepare('SELECT "path", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY "path", chunk');
     const rows = await stmt.all();
     const best = new Map();
     for (const row of rows){
         if (allowed && !allowed.has(row.path)) continue;
         const score = cosineSimilarity(q, decode(row.vector, dims));
         const existing = best.get(row.path);
-        if (!existing || score > existing.score) best.set(row.path, {
-            score,
-            lines: `L${row.start_line}-${row.end_line}`
-        });
+        if (!existing || score > existing.score || score === existing.score && row.chunk < existing.chunk) {
+            best.set(row.path, {
+                score,
+                chunk: row.chunk,
+                lines: `L${row.start_line}-${row.end_line}`
+            });
+        }
     }
     return [
         ...best.entries()
-    ].sort((a, b)=>b[1].score - a[1].score).slice(0, fetch).map(([path, b])=>({
+    ].map(([path, value])=>({
+            path,
+            ...value
+        })).sort(compareVectorScores).slice(0, fetch).map(({ path, lines, score })=>({
             path,
-            lines: b.lines,
-            similarity: asCosine(b.score)
+            lines,
+            similarity: asCosine(score)
         }));
 }
 // Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured
@@ -91,7 +97,10 @@ export async function scanSimilar(conn, dims, path, opts) {
     }
     return [
         ...best.entries()
-    ].sort((a, b)=>b[1] - a[1]).slice(0, opts.k).map(([p, score])=>({
+    ].map(([p, score])=>({
+            path: p,
+            score
+        })).sort(compareVectorScores).slice(0, opts.k).map(({ path: p, score })=>({
             path: p,
             similarity: asCosine(score)
         }));

package/dist/esm/store/turso/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\n\n// Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans\n// score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.\n\n// A model narrower than the column zero-pads: the added dimensions contribute nothing to\n// either dot product or norm, so cosine is unchanged.\nfunction padded(values: ArrayLike<number>, dims: number): Float32Array {\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\n// A bound Float32Array stores truncated bytes with no error; a Buffer view over the same\n// bytes round-trips exact (measured 0.7.2).\nfunction toBlob(v: Float32Array): Buffer {\n  return Buffer.from(v.buffer, v.byteOffset, v.byteLength);\n}\n\nfunction decode(vector: Buffer, dims: number): Float32Array {\n  return new Float32Array(vector.buffer, vector.byteOffset, dims);\n}\n\n// Rows arrive int8-quantized with a per-vector scale (embed/query.ts's toStore); this store\n// dequantizes to floats, as duckdb does.\nfunction dequantize(row: VectorWriteRow, dims: number): Float32Array {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(q.length, dims); d++) out[d] = q[d] * row.scale;\n  return out;\n}\n\nfunction cosineSimilarity(a: Float32Array, b: Float32Array): number {\n  let dot = 0;\n  let na = 0;\n  let nb = 0;\n  for (let d = 0; d < a.length; d++) {\n    dot += a[d] * b[d];\n    na += a[d] * a[d];\n    nb += b[d] * b[d];\n  }\n  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-32);\n}\n\n// `scale` stays NULL: the column exists so the row shape matches the other stores, and\n// cosine is scale-invariant once the value is dequantized here.\nexport async function writeVectorBatch(conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [toBlob(dequantize(row, dims)), row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine. GROUP BY with MIN() cannot do this on 0.7.2: the aggregate\n// is right but the bare start_line/end_line come from an arbitrary row in the group.\nexport async function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const q = padded(qv, dims);\n  const stmt = await conn.prepare('SELECT \"path\", start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; start_line: number; end_line: number; vector: Buffer }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const score = cosineSimilarity(q, decode(row.vector, dims));\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured\n// 6-8x slower here at 26k chunks (2026-08-30).\nexport async function scanSimilar(conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Buffer }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => decode(row.vector, dims));\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Buffer }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = decode(row.vector, dims);\n    for (const t of targets) {\n      const score = cosineSimilarity(t, other);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["asCosine","sampleEvenly","padded","values","dims","out","Float32Array","d","Math","min","length","toBlob","v","Buffer","from","buffer","byteOffset","byteLength","decode","vector","dequantize","row","q","Int8Array","scale","cosineSimilarity","a","b","dot","na","nb","sqrt","writeVectorBatch","conn","rows","runBatch","map","path","chunk","scanCandidates","qv","fetch","allowed","stmt","prepare","all","best","Map","has","score","existing","get","set","lines","start_line","end_line","entries","sort","slice","similarity","scanSimilar","opts","size","targetStmt","targetRows","targets","exclude","other","t","undefined","k","p"],"mappings":"AACA,SAASA,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AAEvD,0FAA0F;AAC1F,wFAAwF;AAExF,yFAAyF;AACzF,sDAAsD;AACtD,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOF;AACT;AAEA,yFAAyF;AACzF,4CAA4C;AAC5C,SAASM,OAAOC,CAAe;IAC7B,OAAOC,OAAOC,IAAI,CAACF,EAAEG,MAAM,EAAEH,EAAEI,UAAU,EAAEJ,EAAEK,UAAU;AACzD;AAEA,SAASC,OAAOC,MAAc,EAAEf,IAAY;IAC1C,OAAO,IAAIE,aAAaa,OAAOJ,MAAM,EAAEI,OAAOH,UAAU,EAAEZ;AAC5D;AAEA,4FAA4F;AAC5F,yCAAyC;AACzC,SAASgB,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,MAAMkB,IAAI,IAAIC,UAAUF,IAAIF,MAAM,CAACJ,MAAM,EAAEM,IAAIF,MAAM,CAACH,UAAU,EAAEK,IAAIF,MAAM,CAACF,UAAU;IACvF,MAAMZ,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACa,EAAEZ,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGe,CAAC,CAACf,EAAE,GAAGc,IAAIG,KAAK;IAC5E,OAAOnB;AACT;AAEA,SAASoB,iBAAiBC,CAAe,EAAEC,CAAe;IACxD,IAAIC,MAAM;IACV,IAAIC,KAAK;IACT,IAAIC,KAAK;IACT,IAAK,IAAIvB,IAAI,GAAGA,IAAImB,EAAEhB,MAAM,EAAEH,IAAK;QACjCqB,OAAOF,CAAC,CAACnB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;QAClBsB,MAAMH,CAAC,CAACnB,EAAE,GAAGmB,CAAC,CAACnB,EAAE;QACjBuB,MAAMH,CAAC,CAACpB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;IACnB;IACA,OAAOqB,MAAOpB,CAAAA,KAAKuB,IAAI,CAACF,MAAMrB,KAAKuB,IAAI,CAACD,MAAM,KAAI;AACpD;AAEA,uFAAuF;AACvF,gEAAgE;AAChE,OAAO,eAAeE,iBAAiBC,IAAgB,EAAE7B,IAAY,EAAE8B,IAAsB;IAC3F,IAAIA,KAAKxB,MAAM,KAAK,GAAG;IACvB,MAAMuB,KAAKE,QAAQ,CACjB,mEACAD,KAAKE,GAAG,CAAC,CAACf,MAAQ;YAACV,OAAOS,WAAWC,KAAKjB;YAAQiB,IAAIgB,IAAI;YAAEhB,IAAIiB,KAAK;SAAC;AAE1E;AAEA,4FAA4F;AAC5F,qFAAqF;AACrF,OAAO,eAAeC,eAAeN,IAAgB,EAAEO,EAAgB,EAAEpC,IAAY,EAAEqC,KAAa,EAAEC,OAAqB;IACzH,MAAMpB,IAAIpB,OAAOsC,IAAIpC;IACrB,MAAMuC,OAAO,MAAMV,KAAKW,OAAO,CAAC;IAChC,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAE5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM1B,OAAOa,KAAM;QACtB,IAAIQ,WAAW,CAACA,QAAQM,GAAG,CAAC3B,IAAIgB,IAAI,GAAG;QACvC,MAAMY,QAAQxB,iBAAiBH,GAAGJ,OAAOG,IAAIF,MAAM,EAAEf;QACrD,MAAM8C,WAAWJ,KAAKK,GAAG,CAAC9B,IAAIgB,IAAI;QAClC,IAAI,CAACa,YAAYD,QAAQC,SAASD,KAAK,EAAEH,KAAKM,GAAG,CAAC/B,IAAIgB,IAAI,EAAE;YAAEY;YAAOI,OAAO,CAAC,CAAC,EAAEhC,IAAIiC,UAAU,CAAC,CAAC,EAAEjC,IAAIkC,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAAC/B,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACsB,KAAK,GAAGvB,CAAC,CAAC,EAAE,CAACuB,KAAK,EACtCS,KAAK,CAAC,GAAGjB,OACTL,GAAG,CAAC,CAAC,CAACC,MAAMV,EAAE,GAAM,CAAA;YAAEU;YAAMgB,OAAO1B,EAAE0B,KAAK;YAAEM,YAAY3D,SAAS2B,EAAEsB,KAAK;QAAE,CAAA;AAC/E;AAEA,6FAA6F;AAC7F,+CAA+C;AAC/C,OAAO,eAAeW,YAAY3B,IAAgB,EAAE7B,IAAY,EAAEiC,IAAY,EAAEwB,IAAgE;IAC9I,IAAIA,KAAKnB,OAAO,IAAImB,KAAKnB,OAAO,CAACoB,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMC,aAAa,MAAM9B,KAAKW,OAAO,CAAC;IACtC,MAAMoB,aAAc,MAAMD,WAAWlB,GAAG,CAACR;IACzC,IAAI2B,WAAWtD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMuD,UAAUhE,aAAa+D,YAAY5B,GAAG,CAAC,CAACf,MAAQH,OAAOG,IAAIF,MAAM,EAAEf;IAEzE,MAAMuC,OAAO,MAAMV,KAAKW,OAAO,CAAC;IAChC,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAC5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM1B,OAAOa,KAAM;QACtB,IAAIb,IAAIgB,IAAI,KAAKA,QAAQwB,KAAKK,OAAO,CAAClB,GAAG,CAAC3B,IAAIgB,IAAI,KAAMwB,KAAKnB,OAAO,IAAI,CAACmB,KAAKnB,OAAO,CAACM,GAAG,CAAC3B,IAAIgB,IAAI,GAAI;QACtG,MAAM8B,QAAQjD,OAAOG,IAAIF,MAAM,EAAEf;QACjC,KAAK,MAAMgE,KAAKH,QAAS;YACvB,MAAMhB,QAAQxB,iBAAiB2C,GAAGD;YAClC,MAAMjB,WAAWJ,KAAKK,GAAG,CAAC9B,IAAIgB,IAAI;YAClC,IAAIa,aAAamB,aAAapB,QAAQC,UAAUJ,KAAKM,GAAG,CAAC/B,IAAIgB,IAAI,EAAEY;QACrE;IACF;IAEA,OAAO;WAAIH,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAAC/B,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BgC,KAAK,CAAC,GAAGG,KAAKS,CAAC,EACflC,GAAG,CAAC,CAAC,CAACmC,GAAGtB,MAAM,GAAM,CAAA;YAAEZ,MAAMkC;YAAGZ,YAAY3D,SAASiD;QAAO,CAAA;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, compareVectorScores, sampleEvenly } from '../vectors.ts';\n\n// Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans\n// score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.\n\n// A model narrower than the column zero-pads: the added dimensions contribute nothing to\n// either dot product or norm, so cosine is unchanged.\nfunction padded(values: ArrayLike<number>, dims: number): Float32Array {\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\n// A bound Float32Array stores truncated bytes with no error; a Buffer view over the same\n// bytes round-trips exact (measured 0.7.2).\nfunction toBlob(v: Float32Array): Buffer {\n  return Buffer.from(v.buffer, v.byteOffset, v.byteLength);\n}\n\nfunction decode(vector: Buffer, dims: number): Float32Array {\n  return new Float32Array(vector.buffer, vector.byteOffset, dims);\n}\n\n// Rows arrive int8-quantized with a per-vector scale (embed/query.ts's toStore); this store\n// dequantizes to floats, as duckdb does.\nfunction dequantize(row: VectorWriteRow, dims: number): Float32Array {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(q.length, dims); d++) out[d] = q[d] * row.scale;\n  return out;\n}\n\nfunction cosineSimilarity(a: Float32Array, b: Float32Array): number {\n  let dot = 0;\n  let na = 0;\n  let nb = 0;\n  for (let d = 0; d < a.length; d++) {\n    dot += a[d] * b[d];\n    na += a[d] * a[d];\n    nb += b[d] * b[d];\n  }\n  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-32);\n}\n\n// `scale` stays NULL: the column exists so the row shape matches the other stores, and\n// cosine is scale-invariant once the value is dequantized here.\nexport async function writeVectorBatch(conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [toBlob(dequantize(row, dims)), row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine. GROUP BY with MIN() cannot do this on 0.7.2: the aggregate\n// is right but the bare start_line/end_line come from an arbitrary row in the group.\nexport async function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const q = padded(qv, dims);\n  const stmt = await conn.prepare('SELECT \"path\", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY \"path\", chunk');\n  const rows = (await stmt.all()) as Array<{ path: string; chunk: number; start_line: number; end_line: number; vector: Buffer }>;\n\n  const best = new Map<string, { score: number; chunk: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const score = cosineSimilarity(q, decode(row.vector, dims));\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score || (score === existing.score && row.chunk < existing.chunk)) {\n      best.set(row.path, { score, chunk: row.chunk, lines: `L${row.start_line}-${row.end_line}` });\n    }\n  }\n  return [...best.entries()]\n    .map(([path, value]) => ({ path, ...value }))\n    .sort(compareVectorScores)\n    .slice(0, fetch)\n    .map(({ path, lines, score }) => ({ path, lines, similarity: asCosine(score) }));\n}\n\n// Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured\n// 6-8x slower here at 26k chunks (2026-08-30).\nexport async function scanSimilar(conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Buffer }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => decode(row.vector, dims));\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Buffer }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = decode(row.vector, dims);\n    for (const t of targets) {\n      const score = cosineSimilarity(t, other);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .map(([p, score]) => ({ path: p, score }))\n    .sort(compareVectorScores)\n    .slice(0, opts.k)\n    .map(({ path: p, score }) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["asCosine","compareVectorScores","sampleEvenly","padded","values","dims","out","Float32Array","d","Math","min","length","toBlob","v","Buffer","from","buffer","byteOffset","byteLength","decode","vector","dequantize","row","q","Int8Array","scale","cosineSimilarity","a","b","dot","na","nb","sqrt","writeVectorBatch","conn","rows","runBatch","map","path","chunk","scanCandidates","qv","fetch","allowed","stmt","prepare","all","best","Map","has","score","existing","get","set","lines","start_line","end_line","entries","value","sort","slice","similarity","scanSimilar","opts","size","targetStmt","targetRows","targets","exclude","other","t","undefined","p","k"],"mappings":"AACA,SAASA,QAAQ,EAAEC,mBAAmB,EAAEC,YAAY,QAAQ,gBAAgB;AAE5E,0FAA0F;AAC1F,wFAAwF;AAExF,yFAAyF;AACzF,sDAAsD;AACtD,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOF;AACT;AAEA,yFAAyF;AACzF,4CAA4C;AAC5C,SAASM,OAAOC,CAAe;IAC7B,OAAOC,OAAOC,IAAI,CAACF,EAAEG,MAAM,EAAEH,EAAEI,UAAU,EAAEJ,EAAEK,UAAU;AACzD;AAEA,SAASC,OAAOC,MAAc,EAAEf,IAAY;IAC1C,OAAO,IAAIE,aAAaa,OAAOJ,MAAM,EAAEI,OAAOH,UAAU,EAAEZ;AAC5D;AAEA,4FAA4F;AAC5F,yCAAyC;AACzC,SAASgB,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,MAAMkB,IAAI,IAAIC,UAAUF,IAAIF,MAAM,CAACJ,MAAM,EAAEM,IAAIF,MAAM,CAACH,UAAU,EAAEK,IAAIF,MAAM,CAACF,UAAU;IACvF,MAAMZ,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACa,EAAEZ,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGe,CAAC,CAACf,EAAE,GAAGc,IAAIG,KAAK;IAC5E,OAAOnB;AACT;AAEA,SAASoB,iBAAiBC,CAAe,EAAEC,CAAe;IACxD,IAAIC,MAAM;IACV,IAAIC,KAAK;IACT,IAAIC,KAAK;IACT,IAAK,IAAIvB,IAAI,GAAGA,IAAImB,EAAEhB,MAAM,EAAEH,IAAK;QACjCqB,OAAOF,CAAC,CAACnB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;QAClBsB,MAAMH,CAAC,CAACnB,EAAE,GAAGmB,CAAC,CAACnB,EAAE;QACjBuB,MAAMH,CAAC,CAACpB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;IACnB;IACA,OAAOqB,MAAOpB,CAAAA,KAAKuB,IAAI,CAACF,MAAMrB,KAAKuB,IAAI,CAACD,MAAM,KAAI;AACpD;AAEA,uFAAuF;AACvF,gEAAgE;AAChE,OAAO,eAAeE,iBAAiBC,IAAgB,EAAE7B,IAAY,EAAE8B,IAAsB;IAC3F,IAAIA,KAAKxB,MAAM,KAAK,GAAG;IACvB,MAAMuB,KAAKE,QAAQ,CACjB,mEACAD,KAAKE,GAAG,CAAC,CAACf,MAAQ;YAACV,OAAOS,WAAWC,KAAKjB;YAAQiB,IAAIgB,IAAI;YAAEhB,IAAIiB,KAAK;SAAC;AAE1E;AAEA,4FAA4F;AAC5F,qFAAqF;AACrF,OAAO,eAAeC,eAAeN,IAAgB,EAAEO,EAAgB,EAAEpC,IAAY,EAAEqC,KAAa,EAAEC,OAAqB;IACzH,MAAMpB,IAAIpB,OAAOsC,IAAIpC;IACrB,MAAMuC,OAAO,MAAMV,KAAKW,OAAO,CAAC;IAChC,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAE5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM1B,OAAOa,KAAM;QACtB,IAAIQ,WAAW,CAACA,QAAQM,GAAG,CAAC3B,IAAIgB,IAAI,GAAG;QACvC,MAAMY,QAAQxB,iBAAiBH,GAAGJ,OAAOG,IAAIF,MAAM,EAAEf;QACrD,MAAM8C,WAAWJ,KAAKK,GAAG,CAAC9B,IAAIgB,IAAI;QAClC,IAAI,CAACa,YAAYD,QAAQC,SAASD,KAAK,IAAKA,UAAUC,SAASD,KAAK,IAAI5B,IAAIiB,KAAK,GAAGY,SAASZ,KAAK,EAAG;YACnGQ,KAAKM,GAAG,CAAC/B,IAAIgB,IAAI,EAAE;gBAAEY;gBAAOX,OAAOjB,IAAIiB,KAAK;gBAAEe,OAAO,CAAC,CAAC,EAAEhC,IAAIiC,UAAU,CAAC,CAAC,EAAEjC,IAAIkC,QAAQ,EAAE;YAAC;QAC5F;IACF;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBpB,GAAG,CAAC,CAAC,CAACC,MAAMoB,MAAM,GAAM,CAAA;YAAEpB;YAAM,GAAGoB,KAAK;QAAC,CAAA,GACzCC,IAAI,CAAC1D,qBACL2D,KAAK,CAAC,GAAGlB,OACTL,GAAG,CAAC,CAAC,EAAEC,IAAI,EAAEgB,KAAK,EAAEJ,KAAK,EAAE,GAAM,CAAA;YAAEZ;YAAMgB;YAAOO,YAAY7D,SAASkD;QAAO,CAAA;AACjF;AAEA,6FAA6F;AAC7F,+CAA+C;AAC/C,OAAO,eAAeY,YAAY5B,IAAgB,EAAE7B,IAAY,EAAEiC,IAAY,EAAEyB,IAAgE;IAC9I,IAAIA,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACqB,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMC,aAAa,MAAM/B,KAAKW,OAAO,CAAC;IACtC,MAAMqB,aAAc,MAAMD,WAAWnB,GAAG,CAACR;IACzC,IAAI4B,WAAWvD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUjE,aAAagE,YAAY7B,GAAG,CAAC,CAACf,MAAQH,OAAOG,IAAIF,MAAM,EAAEf;IAEzE,MAAMuC,OAAO,MAAMV,KAAKW,OAAO,CAAC;IAChC,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAC5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM1B,OAAOa,KAAM;QACtB,IAAIb,IAAIgB,IAAI,KAAKA,QAAQyB,KAAKK,OAAO,CAACnB,GAAG,CAAC3B,IAAIgB,IAAI,KAAMyB,KAAKpB,OAAO,IAAI,CAACoB,KAAKpB,OAAO,CAACM,GAAG,CAAC3B,IAAIgB,IAAI,GAAI;QACtG,MAAM+B,QAAQlD,OAAOG,IAAIF,MAAM,EAAEf;QACjC,KAAK,MAAMiE,KAAKH,QAAS;YACvB,MAAMjB,QAAQxB,iBAAiB4C,GAAGD;YAClC,MAAMlB,WAAWJ,KAAKK,GAAG,CAAC9B,IAAIgB,IAAI;YAClC,IAAIa,aAAaoB,aAAarB,QAAQC,UAAUJ,KAAKM,GAAG,CAAC/B,IAAIgB,IAAI,EAAEY;QACrE;IACF;IAEA,OAAO;WAAIH,KAAKU,OAAO;KAAG,CACvBpB,GAAG,CAAC,CAAC,CAACmC,GAAGtB,MAAM,GAAM,CAAA;YAAEZ,MAAMkC;YAAGtB;QAAM,CAAA,GACtCS,IAAI,CAAC1D,qBACL2D,KAAK,CAAC,GAAGG,KAAKU,CAAC,EACfpC,GAAG,CAAC,CAAC,EAAEC,MAAMkC,CAAC,EAAEtB,KAAK,EAAE,GAAM,CAAA;YAAEZ,MAAMkC;YAAGX,YAAY7D,SAASkD;QAAO,CAAA;AACzE"}

package/dist/esm/store/types.d.ts

@@ -1,7 +1,7 @@
 import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
-export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "lexical", "vectors", "sql-functions"];
+export declare const CAPABILITY_NAMES: readonly ["vectors", "segment"];
 export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;
@@ -55,7 +55,6 @@ export interface DocumentStore {
 }
 export interface LexicalHit {
     path: string;
-    hit: string | null;
 }
 export interface LexicalQueryOptions {
     whereJoin: string;

package/dist/esm/store/types.js

@@ -1,13 +1,11 @@
 // The backing-store interface: a minimal portable statement surface (exec/prepare), plus
 // dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).
-// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not
-// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,
-// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open
-// or first use. An array, not a bare union, so a runtime check reads the same list the type does.
+// 'vectors': the store's VectorStore is functionally implemented, not present-but-inert; checked
+// at open (index.ts) before a config with an "embed" block is allowed onto a store lacking it.
+// 'segment': the engine can run segment() (Intl.Segmenter grapheme clustering) as a SQL function;
+// turso's client cannot register it and has no rewrite for it, unlike has()/basename(), which now
+// work everywhere. An array, not a bare union, so a runtime check reads the same list the type does.
 export const CAPABILITY_NAMES = [
-    'phrases',
-    'snippets',
-    'lexical',
     'vectors',
-    'sql-functions'
+    'segment'
 ];

package/dist/esm/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,6FAA6F;AAC7F,mGAAmG;AACnG,mGAAmG;AACnG,kGAAkG;AAClG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB,CAAU"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'vectors': the store's VectorStore is functionally implemented, not present-but-inert; checked\n// at open (index.ts) before a config with an \"embed\" block is allowed onto a store lacking it.\n// 'segment': the engine can run segment() (Intl.Segmenter grapheme clustering) as a SQL function;\n// turso's client cannot register it and has no rewrite for it, unlike has()/basename(), which now\n// work everywhere. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['vectors', 'segment'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\n// A ranked match, nothing more. Snippets are cut from the document above the store\n// (commands/search.ts), identically for every engine, so no store carries them.\nexport interface LexicalHit {\n  path: string;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with snippet, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,iGAAiG;AACjG,+FAA+F;AAC/F,kGAAkG;AAClG,kGAAkG;AAClG,qGAAqG;AACrG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;CAAU,CAAU"}

package/dist/esm/store/vectors.d.ts

@@ -2,6 +2,13 @@ import type { Connection } from './types.js';
 export declare const TARGET_CHUNK_CAP = 16;
 export declare function sampleEvenly<T>(rows: T[], cap?: number): T[];
 export declare function asCosine(score: number): number;
+export declare function compareVectorScores(a: {
+    path: string;
+    score: number;
+}, b: {
+    path: string;
+    score: number;
+}): number;
 export declare function pendingRows(conn: Connection): Promise<Array<{
     path: string;
     chunk: number;

package/dist/esm/store/vectors.js

@@ -7,11 +7,17 @@ export function sampleEvenly(rows, cap = TARGET_CHUNK_CAP) {
     const step = Math.max(1, Math.ceil(rows.length / cap));
     return rows.filter((_, i)=>i % step === 0);
 }
-// Dequantised int8 dot products (sqlite) land a little either side of a true cosine (an identical
-// pair can print 1.001); array_cosine_similarity (duckdb) has no such error but is rounded the same way, so both stores print the same bounded number.
+// Every store computes a real cosine; clamp float error at the ends and round for the public score.
 export function asCosine(score) {
     return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
 }
+// Rank by the computed cosine before rounding for display. Bytewise path order resolves only an
+// exact score tie, so two distinguishable similarities never cross a caller's result boundary.
+export function compareVectorScores(a, b) {
+    const byScore = b.score - a.score;
+    if (byScore !== 0) return byScore;
+    return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));
+}
 export async function pendingRows(conn) {
     const stmt = await conn.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk');
     return await stmt.all();

package/dist/esm/store/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Dequantised int8 dot products (sqlite) land a little either side of a true cosine (an identical\n// pair can print 1.001); array_cosine_similarity (duckdb) has no such error but is rounded the same way, so both stores print the same bounded number.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","pendingRows","conn","stmt","prepare","all","hasVectorRow","path","row","get","undefined"],"mappings":"AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMA,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,kGAAkG;AAClG,uJAAuJ;AACvJ,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,OAAO,eAAeG,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEK,IAAY;IAC/D,MAAMJ,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMI,MAAO,MAAML,KAAKM,GAAG,CAACF;IAC5B,OAAOC,QAAQE;AACjB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Every store computes a real cosine; clamp float error at the ends and round for the public score.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Rank by the computed cosine before rounding for display. Bytewise path order resolves only an\n// exact score tie, so two distinguishable similarities never cross a caller's result boundary.\nexport function compareVectorScores(a: { path: string; score: number }, b: { path: string; score: number }): number {\n  const byScore = b.score - a.score;\n  if (byScore !== 0) return byScore;\n  return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","compareVectorScores","a","b","byScore","Buffer","compare","from","path","pendingRows","conn","stmt","prepare","all","hasVectorRow","row","get","undefined"],"mappings":"AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMA,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,oGAAoG;AACpG,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,OAAO,SAASG,oBAAoBC,CAAkC,EAAEC,CAAkC;IACxG,MAAMC,UAAUD,EAAEL,KAAK,GAAGI,EAAEJ,KAAK;IACjC,IAAIM,YAAY,GAAG,OAAOA;IAC1B,OAAOC,OAAOC,OAAO,CAACD,OAAOE,IAAI,CAACL,EAAEM,IAAI,GAAGH,OAAOE,IAAI,CAACJ,EAAEK,IAAI;AAC/D;AAEA,OAAO,eAAeC,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEF,IAAY;IAC/D,MAAMG,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMG,MAAO,MAAMJ,KAAKK,GAAG,CAACR;IAC5B,OAAOO,QAAQE;AACjB"}

package/dist/esm/text/segment.d.ts

@@ -1,4 +1,17 @@
 export declare const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}";
 export declare function hasUnspacedRun(text: string): boolean;
+export interface SearchToken {
+    text: string;
+    start: number;
+    end: number;
+    unspaced: boolean;
+}
+export declare function searchTokens(text: string): SearchToken[];
+export declare function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean;
+export declare function unspacedRuns(text: string): Array<{
+    start: number;
+    end: number;
+}>;
 export declare function segmentField(text: string): string;
 export declare function segmentMatch(terms: string): string;
+export declare function foldForSearch(text: string): string;