sensemaking 0.23.0 → 0.24.0

package/dist/esm/store/turso/connection.js

@@ -1,4 +1,9 @@
+import { stat } from 'node:fs/promises';
+import { rewriteInsert } from '../batch.js';
+import { setMeta } from '../shared.js';
 import { BEGIN_WRITE, withTransaction } from '../transaction.js';
+import { CONNECT_OPTS, tursoApi } from './native.js';
+import { rewriteFunctions } from './sql-functions.js';
 // @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%
 // faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.
 let TursoStatementWrapper = class TursoStatementWrapper {
@@ -33,22 +38,73 @@ export async function checkpointWal(db) {
         console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${err.message}`);
     }
 }
+// The main cache file's path, or null when unresolvable. The store holds a Database and
+// Connection, not a path (PLAN 3.52), so the path is read from the database itself.
+export async function cacheFilePath(conn) {
+    var _rows_find;
+    const rows = await (await conn.prepare('PRAGMA database_list')).all();
+    return ((_rows_find = rows.find((r)=>r.name === 'main')) === null || _rows_find === void 0 ? void 0 : _rows_find.file) || null;
+}
+export async function fileSize(path) {
+    try {
+        return (await stat(path)).size;
+    } catch  {
+        return null;
+    }
+}
+// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the
+// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).
+export async function reclaimSpace(path) {
+    try {
+        const turso = await tursoApi();
+        const db = await turso.connect(path, {
+            ...CONNECT_OPTS,
+            experimental: [
+                ...CONNECT_OPTS.experimental,
+                'vacuum'
+            ]
+        });
+        try {
+            await db.exec('VACUUM');
+            const size = await fileSize(path);
+            // Recorded here, not by the caller: this connection is the only one still open on the file.
+            if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));
+            await checkpointWal(db);
+        } finally{
+            await db.close();
+        }
+    } catch (err) {
+        console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${err.message}`);
+    }
+}
 export function createConnection(db) {
     const conn = {
         async exec (sql) {
             await db.exec(sql);
         },
         async prepare (sql) {
-            return new TursoStatementWrapper(await db.prepare(sql));
+            return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));
         },
-        // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as
-        // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.
+        // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):
+        // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.
         async runBatch (sql, paramRows) {
             if (paramRows.length === 0) return;
             await withTransaction(conn, async ()=>{
-                const stmt = await db.prepare(sql);
+                const rewritten = rewriteInsert(sql, paramRows.length);
+                if (rewritten) {
+                    const stmt = await db.prepare(rewritten.sql);
+                    try {
+                        // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack
+                        // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).
+                        await stmt.run(paramRows.flat());
+                    } finally{
+                        await stmt.close();
+                    }
+                    return;
+                }
                 // Finalized here because nothing else will: open() hands back a connection the caller
-                // can hold across many batches, and the db.batch() this replaced finalized its own.
+                // can hold across many batches, and nothing else owns this statement's lifetime.
+                const stmt = await db.prepare(sql);
                 try {
                     for (const row of paramRows)await stmt.run(...row);
                 } finally{

package/dist/esm/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as\n    // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = await db.prepare(sql);\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","createConnection","conn","sql","prepare","runBatch","paramRows","length","row","close"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAOjE,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,OAAO,SAASC,iBAAiBN,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKO,GAAW;YACpB,MAAMR,GAAGC,IAAI,CAACO;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIpB,sBAAsB,MAAMY,GAAGS,OAAO,CAACD;QACpD;QACA,+FAA+F;QAC/F,wKAAwK;QACxK,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMzB,gBACJoB,MACA;gBACE,MAAMhB,OAAO,MAAMS,GAAGS,OAAO,CAACD;gBAC9B,sFAAsF;gBACtF,oFAAoF;gBACpF,IAAI;oBACF,KAAK,MAAMK,OAAOF,UAAW,MAAMpB,KAAKF,GAAG,IAAIwB;gBACjD,SAAU;oBACR,MAAMtB,KAAKuB,KAAK;gBAClB;YACF,GACA5B;QAEJ;IACF;IACA,OAAOqB;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and nothing else owns this statement's lifetime.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["stat","rewriteInsert","setMeta","BEGIN_WRITE","withTransaction","CONNECT_OPTS","tursoApi","rewriteFunctions","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","cacheFilePath","conn","rows","prepare","find","r","name","file","fileSize","path","size","reclaimSpace","turso","connect","experimental","createConnection","String","close","sql","runBatch","paramRows","length","rewritten","flat","row"],"mappings":"AAAA,SAASA,IAAI,QAAQ,mBAAmB;AAExC,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,YAAY,EAAEC,QAAQ,QAAQ,cAAc;AACrD,SAASC,gBAAgB,QAAQ,qBAAqB;AAMtD,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,eAAeC,cAAcC,IAAgB;QAE3CC;IADP,MAAMA,OAAQ,MAAM,AAAC,CAAA,MAAMD,KAAKE,OAAO,CAAC,uBAAsB,EAAGhB,GAAG;IACpE,OAAOe,EAAAA,aAAAA,KAAKE,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,qBAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;AACtD;AAEA,OAAO,eAAeC,SAASC,IAAY;IACzC,IAAI;QACF,OAAO,AAAC,CAAA,MAAMnC,KAAKmC,KAAI,EAAGC,IAAI;IAChC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,iGAAiG;AACjG,sGAAsG;AACtG,OAAO,eAAeC,aAAaF,IAAY;IAC7C,IAAI;QACF,MAAMG,QAAQ,MAAMhC;QACpB,MAAMc,KAAK,MAAMkB,MAAMC,OAAO,CAACJ,MAAM;YAAE,GAAG9B,YAAY;YAAEmC,cAAc;mBAAInC,aAAamC,YAAY;gBAAE;aAAS;QAAC;QAC/G,IAAI;YACF,MAAMpB,GAAGC,IAAI,CAAC;YACd,MAAMe,OAAO,MAAMF,SAASC;YAC5B,4FAA4F;YAC5F,IAAIC,SAAS,MAAM,MAAMlC,QAAQuC,iBAAiBrB,KAAK,gBAAgBsB,OAAON;YAC9E,MAAMjB,cAAcC;QACtB,SAAU;YACR,MAAMA,GAAGuB,KAAK;QAChB;IACF,EAAE,OAAOrB,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,8EAA8E,EAAE,AAACF,IAAcG,OAAO,EAAE;IACzH;AACF;AAEA,OAAO,SAASgB,iBAAiBrB,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKuB,GAAW;YACpB,MAAMxB,GAAGC,IAAI,CAACuB;QAChB;QACA,MAAMf,SAAQe,GAAW;YACvB,OAAO,IAAIpC,sBAAsB,MAAMY,GAAGS,OAAO,CAACtB,iBAAiBqC;QACrE;QACA,gGAAgG;QAChG,wGAAwG;QACxG,MAAMC,UAASD,GAAW,EAAEE,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAM3C,gBACJuB,MACA;gBACE,MAAMqB,YAAY/C,cAAc2C,KAAKE,UAAUC,MAAM;gBACrD,IAAIC,WAAW;oBACb,MAAMrC,OAAO,MAAMS,GAAGS,OAAO,CAACmB,UAAUJ,GAAG;oBAC3C,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,MAAMjC,KAAKF,GAAG,CAACqC,UAAUG,IAAI;oBAC/B,SAAU;wBACR,MAAMtC,KAAKgC,KAAK;oBAClB;oBACA;gBACF;gBACA,sFAAsF;gBACtF,iFAAiF;gBACjF,MAAMhC,OAAO,MAAMS,GAAGS,OAAO,CAACe;gBAC9B,IAAI;oBACF,KAAK,MAAMM,OAAOJ,UAAW,MAAMnC,KAAKF,GAAG,IAAIyC;gBACjD,SAAU;oBACR,MAAMvC,KAAKgC,KAAK;gBAClB;YACF,GACAxC;QAEJ;IACF;IACA,OAAOwB;AACT"}

package/dist/esm/store/turso/lexical-text.d.ts

@@ -0,0 +1 @@
+export declare function stemFolded(text: string): string;

package/dist/esm/store/turso/lexical-text.js

@@ -0,0 +1,12 @@
+import { stemmer } from 'stemmer';
+import { foldForSearch, searchTokens } from '../../text/segment.js';
+export function stemFolded(text) {
+    let out = '';
+    let end = 0;
+    for (const token of searchTokens(text)){
+        out += text.slice(end, token.start).replace(/[\u0027\u2019]/gu, ' ');
+        out += token.unspaced ? token.text : stemmer(foldForSearch(token.text));
+        end = token.end;
+    }
+    return out + text.slice(end).replace(/[\u0027\u2019]/gu, ' ');
+}

package/dist/esm/store/turso/lexical-text.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical-text.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\nimport { foldForSearch, searchTokens } from '../../text/segment.ts';\n\nexport function stemFolded(text: string): string {\n  let out = '';\n  let end = 0;\n  for (const token of searchTokens(text)) {\n    out += text.slice(end, token.start).replace(/[\\u0027\\u2019]/gu, ' ');\n    out += token.unspaced ? token.text : stemmer(foldForSearch(token.text));\n    end = token.end;\n  }\n  return out + text.slice(end).replace(/[\\u0027\\u2019]/gu, ' ');\n}\n"],"names":["stemmer","foldForSearch","searchTokens","stemFolded","text","out","end","token","slice","start","replace","unspaced"],"mappings":"AAAA,SAASA,OAAO,QAAQ,UAAU;AAClC,SAASC,aAAa,EAAEC,YAAY,QAAQ,wBAAwB;AAEpE,OAAO,SAASC,WAAWC,IAAY;IACrC,IAAIC,MAAM;IACV,IAAIC,MAAM;IACV,KAAK,MAAMC,SAASL,aAAaE,MAAO;QACtCC,OAAOD,KAAKI,KAAK,CAACF,KAAKC,MAAME,KAAK,EAAEC,OAAO,CAAC,oBAAoB;QAChEL,OAAOE,MAAMI,QAAQ,GAAGJ,MAAMH,IAAI,GAAGJ,QAAQC,cAAcM,MAAMH,IAAI;QACrEE,MAAMC,MAAMD,GAAG;IACjB;IACA,OAAOD,MAAMD,KAAKI,KAAK,CAACF,KAAKI,OAAO,CAAC,oBAAoB;AAC3D"}

package/dist/esm/store/turso/lexical.js

@@ -1,17 +1,23 @@
 // T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and
 // quoted phrases, plus a second ngram-tokenized index over "_ngram" sidecars for unspaced-script runs.
 import { SenseError } from '../../errors.js';
-import { hasUnspacedRun } from '../../text/segment.js';
+import { matchesSearchPhrase, searchTokens } from '../../text/segment.js';
+import { stemFolded } from './lexical-text.js';
 const FIELDS = [
-    'title',
-    'summary',
-    'text'
+    'title_stem',
+    'summary_stem',
+    'text_stem'
 ];
 const NGRAM_FIELDS = [
     'title_ngram',
     'summary_ngram',
     'text_ngram'
 ];
+const AUTHORED_FIELDS = [
+    'title',
+    'summary',
+    'text'
+];
 // FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:
 // no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.
 const FTS5_OPERATORS = [
@@ -53,22 +59,30 @@ function splitTerms(terms) {
     const words = [];
     const phrases = [];
     const unspaced = [];
+    let emptyPhrase = false;
     const withoutPhrases = terms.replace(/"([^"]*)"/g, (_m, inner)=>{
         const phrase = inner.trim();
-        if (phrase.length === 0) return ' ';
-        if (hasUnspacedRun(phrase)) unspaced.push(phrase);
-        else phrases.push(phrase);
+        const tokens = searchTokens(phrase);
+        if (tokens.length === 0) {
+            emptyPhrase = true;
+            return ' ';
+        }
+        unspaced.push(...tokens.filter((token)=>token.unspaced).map((token)=>token.text));
+        phrases.push(tokens);
         return ' ';
     });
     for (const tok of withoutPhrases.split(/\s+/)){
         if (tok.length === 0) continue;
-        if (hasUnspacedRun(tok)) unspaced.push(tok);
-        else words.push(tok);
+        for (const token of searchTokens(tok)){
+            if (token.unspaced) unspaced.push(token.text);
+            else words.push(token.text);
+        }
     }
     return {
         words,
         phrases,
-        unspaced
+        unspaced,
+        emptyPhrase
     };
 }
 // fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays
@@ -79,9 +93,16 @@ function escapeFtsLiteral(query) {
 // Terms are quoted ("O'Brien's" is a parse error bare) and AND-joined: Tantivy's default for
 // bare terms is disjunctive, so space-joining would loosen "apple banana" into "apple OR banana".
 function buildQueries(words, phrases, unspaced) {
+    const phraseParts = phrases.flatMap((phrase)=>{
+        const latin = phrase.filter((token)=>!token.unspaced).map((token)=>stemFolded(token.text));
+        if (latin.length === 0) return [];
+        return phrase.some((token)=>token.unspaced) ? latin : [
+            latin.join(' ')
+        ];
+    });
     const defaultParts = [
-        ...words,
-        ...phrases
+        ...words.map(stemFolded),
+        ...phraseParts
     ].map((t)=>`"${t}"`);
     const unspacedParts = unspaced.map((t)=>`"${t}"`);
     return {
@@ -98,31 +119,45 @@ function ftsBranch(cols, query, opts) {
     FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;
 }
 // Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's
-// queryLexical take). `hit` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS excerpt fallback (commands/search.ts).
+// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).
 export async function queryLexical(conn, terms, opts) {
     const unsupported = unsupportedOperator(terms);
     if (unsupported !== null) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "turso" does not implement FTS5's ${unsupported.label} ("${unsupported.token}") in this build; rephrase "${terms.trim()}" without it, or set "store" to "sqlite" in this tree's config to search it as written`);
     }
-    const { words, phrases, unspaced } = splitTerms(terms);
+    const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);
+    if (emptyPhrase) return [];
     const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);
     if (defaultQuery === '' && ngramQuery === '') return [];
     const branches = [];
     if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));
     if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));
     // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,
-    // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.
-    const sql = `SELECT path, NULL AS hit FROM (
-    SELECT path, SUM(score) AS score FROM (
+    // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase
+    // candidates carry authored fields through the join below so the shared verifier can reject
+    // cross-field, reordered, and non-adjacent matches before applying the caller's limit.
+    const ranked = `SELECT path, SUM(score) AS score FROM (
       ${branches.join(' UNION ALL ')}
-    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}
-  ) sq ORDER BY score DESC, path LIMIT ?`;
+    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;
+    const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
+    const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';
+    const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (
+    ${ranked}
+  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;
     const stmt = await conn.prepare(sql);
-    const rows = await stmt.all(opts.limit);
+    const rows = await stmt.all(...phrases.length > 0 ? [] : [
+        opts.limit
+    ]);
     // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,
     // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).
     if (rows.some((r)=>r.path === null)) {
         throw new SenseError('LEXICAL_NULL_PATH', 'store "turso" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');
     }
-    return rows;
+    const matches = phrases.length === 0 ? rows : rows.filter((row)=>phrases.every((phrase)=>AUTHORED_FIELDS.some((field)=>{
+                var _row_field;
+                return matchesSearchPhrase((_row_field = row[field]) !== null && _row_field !== void 0 ? _row_field : '', phrase);
+            })));
+    return matches.slice(0, opts.limit).map(({ path })=>({
+            path
+        }));
 }

package/dist/esm/store/turso/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: string[]; unspaced: string[] } {\n  const words: string[] = [];\n  const phrases: string[] = [];\n  const unspaced: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    if (hasUnspacedRun(phrase)) unspaced.push(phrase);\n    else phrases.push(phrase);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) unspaced.push(tok);\n    else words.push(tok);\n  }\n  return { words, phrases, unspaced };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: string[], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const defaultParts = [...words, ...phrases].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `hit` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced } = splitTerms(terms);\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(opts.limit)) as unknown as LexicalHit[];\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  return rows;\n}\n"],"names":["SenseError","hasUnspacedRun","FIELDS","NGRAM_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","_m","inner","phrase","length","push","tok","split","escapeFtsLiteral","query","buildQueries","defaultParts","map","t","unspacedParts","defaultQuery","join","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","queryLexical","conn","unsupported","branches","sql","stmt","prepare","rows","all","limit","some","r","path"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;AACvG,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AAGvD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAC3C,MAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AAEnE,uFAAuF;AACvF,+FAA+F;AAC/F,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,UAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMT,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACS,IAAIC;QACtD,MAAMC,SAASD,MAAMN,IAAI;QACzB,IAAIO,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChC,IAAIrB,eAAeoB,SAASH,SAASK,IAAI,CAACF;aACrCJ,QAAQM,IAAI,CAACF;QAClB,OAAO;IACT;IACA,KAAK,MAAMG,OAAOf,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIF,MAAM,KAAK,GAAG;QACtB,IAAIrB,eAAeuB,MAAMN,SAASK,IAAI,CAACC;aAClCR,MAAMO,IAAI,CAACC;IAClB;IACA,OAAO;QAAER;QAAOC;QAASC;IAAS;AACpC;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASQ,iBAAiBC,KAAa;IACrC,OAAOA,MAAMjB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASkB,aAAaZ,KAAe,EAAEC,OAAiB,EAAEC,QAAkB;IAC1E,MAAMW,eAAe;WAAIb;WAAUC;KAAQ,CAACa,GAAG,CAAC,CAACC,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAC/D,MAAMC,gBAAgBd,SAASY,GAAG,CAAC,CAACC,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAClD,OAAO;QAAEE,cAAcJ,aAAaK,IAAI,CAAC;QAAUC,YAAYH,cAAcE,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASE,UAAUC,IAAuB,EAAEV,KAAa,EAAEW,IAAyB;IAClF,MAAMC,UAAUF,KAAKP,GAAG,CAAC,CAACU,IAAM,CAAC,QAAQ,EAAEA,GAAG,EAAEN,IAAI,CAAC;IACrD,MAAMO,MAAMf,iBAAiBC;IAC7B,OAAO,CAAC,yCAAyC,EAAEY,QAAQ,GAAG,EAAEE,IAAI;iBACrD,EAAEH,KAAKI,SAAS,CAAC,iBAAiB,EAAEH,QAAQ,GAAG,EAAEE,IAAI,GAAG,EAAEH,KAAKK,SAAS,CAAC,CAAC,EAAEL,KAAKM,SAAS,EAAE;AAC7G;AAEA,uFAAuF;AACvF,gMAAgM;AAChM,OAAO,eAAeC,aAAaC,IAAgB,EAAEtC,KAAa,EAAE8B,IAAyB;IAC3F,MAAMS,cAAcxC,oBAAoBC;IACxC,IAAIuC,gBAAgB,MAAM;QACxB,MAAM,IAAI/C,WAAW,4BAA4B,CAAC,wCAAwC,EAAE+C,YAAY1C,KAAK,CAAC,GAAG,EAAE0C,YAAYlC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IACzQ;IACA,MAAM,EAAEE,KAAK,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGH,WAAWP;IAChD,MAAM,EAAEyB,YAAY,EAAEE,UAAU,EAAE,GAAGP,aAAaZ,OAAOC,SAASC;IAClE,IAAIe,iBAAiB,MAAME,eAAe,IAAI,OAAO,EAAE;IAEvD,MAAMa,WAAqB,EAAE;IAC7B,IAAIf,iBAAiB,IAAIe,SAASzB,IAAI,CAACa,UAAUlC,QAAQ+B,cAAcK;IACvE,IAAIH,eAAe,IAAIa,SAASzB,IAAI,CAACa,UAAUjC,cAAcgC,YAAYG;IAEzE,8FAA8F;IAC9F,mFAAmF;IACnF,MAAMW,MAAM,CAAC;;MAET,EAAED,SAASd,IAAI,CAAC,eAAe;4CACO,EAAEc,SAAS1B,MAAM,CAAC;wCACtB,CAAC;IACvC,MAAM4B,OAAO,MAAMJ,KAAKK,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,CAACf,KAAKgB,KAAK;IACvC,+FAA+F;IAC/F,yFAAyF;IACzF,IAAIF,KAAKG,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,OAAO;QACrC,MAAM,IAAIzD,WAAW,qBAAqB;IAC5C;IACA,OAAOoD;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const phrases: SearchToken[][] = [];\n  const unspaced: string[] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const tokens = searchTokens(phrase);\n    if (tokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n    phrases.push(tokens);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) unspaced.push(token.text);\n      else words.push(token.text);\n    }\n  }\n  return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const phraseParts = phrases.flatMap((phrase) => {\n    const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n    if (latin.length === 0) return [];\n    return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n  });\n  const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase) return [];\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n  // candidates carry authored fields through the join below so the shared verifier can reject\n  // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n  const ranked = `SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n  const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n  const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n    ${ranked}\n  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["SenseError","matchesSearchPhrase","searchTokens","stemFolded","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","length","push","filter","map","text","tok","split","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","queryLexical","conn","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","stmt","prepare","rows","all","limit","r","path","matches","row","every","field","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;AACvG,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAE5F,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,MAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,MAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,MAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,UAA2B,EAAE;IACnC,MAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,MAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACU,IAAIC;QACtD,MAAMC,SAASD,MAAMP,IAAI;QACzB,MAAMS,SAASxB,aAAauB;QAC5B,IAAIC,OAAOC,MAAM,KAAK,GAAG;YACvBL,cAAc;YACd,OAAO;QACT;QACAD,SAASO,IAAI,IAAIF,OAAOG,MAAM,CAAC,CAACb,QAAUA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUA,MAAMe,IAAI;QACnFX,QAAQQ,IAAI,CAACF;QACb,OAAO;IACT;IACA,KAAK,MAAMM,OAAOpB,eAAeqB,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIL,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMX,SAASd,aAAa8B,KAAM;YACrC,IAAIhB,MAAMK,QAAQ,EAAEA,SAASO,IAAI,CAACZ,MAAMe,IAAI;iBACvCZ,MAAMS,IAAI,CAACZ,MAAMe,IAAI;QAC5B;IACF;IACA,OAAO;QAAEZ;QAAOC;QAASC;QAAUC;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASY,iBAAiBC,KAAa;IACrC,OAAOA,MAAMtB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASuB,aAAajB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,MAAMgB,cAAcjB,QAAQkB,OAAO,CAAC,CAACb;QACnC,MAAMc,QAAQd,OAAOI,MAAM,CAAC,CAACb,QAAU,CAACA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUb,WAAWa,MAAMe,IAAI;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOF,OAAOe,IAAI,CAAC,CAACxB,QAAUA,MAAMK,QAAQ,IAAIkB,QAAQ;YAACA,MAAME,IAAI,CAAC;SAAK;IAC3E;IACA,MAAMC,eAAe;WAAIvB,MAAMW,GAAG,CAAC3B;WAAgBkC;KAAY,CAACP,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IACnF,MAAMC,gBAAgBvB,SAASS,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAClD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEb,KAAa,EAAEc,IAAyB;IAClF,MAAMC,UAAUF,KAAKlB,GAAG,CAAC,CAACqB,IAAM,CAAC,QAAQ,EAAEA,GAAG,EAAEV,IAAI,CAAC;IACrD,MAAMW,MAAMlB,iBAAiBC;IAC7B,OAAO,CAAC,yCAAyC,EAAEe,QAAQ,GAAG,EAAEE,IAAI;iBACrD,EAAEH,KAAKI,SAAS,CAAC,iBAAiB,EAAEH,QAAQ,GAAG,EAAEE,IAAI,GAAG,EAAEH,KAAKK,SAAS,CAAC,CAAC,EAAEL,KAAKM,SAAS,EAAE;AAC7G;AAEA,uFAAuF;AACvF,qMAAqM;AACrM,OAAO,eAAeC,aAAaC,IAAgB,EAAE9C,KAAa,EAAEsC,IAAyB;IAC3F,MAAMS,cAAchD,oBAAoBC;IACxC,IAAI+C,gBAAgB,MAAM;QACxB,MAAM,IAAI1D,WAAW,4BAA4B,CAAC,wCAAwC,EAAE0D,YAAYlD,KAAK,CAAC,GAAG,EAAEkD,YAAY1C,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IACzQ;IACA,MAAM,EAAEE,KAAK,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ,WAAWP;IAC7D,IAAIW,aAAa,OAAO,EAAE;IAC1B,MAAM,EAAEuB,YAAY,EAAEC,UAAU,EAAE,GAAGV,aAAajB,OAAOC,SAASC;IAClE,IAAIwB,iBAAiB,MAAMC,eAAe,IAAI,OAAO,EAAE;IAEvD,MAAMa,WAAqB,EAAE;IAC7B,IAAId,iBAAiB,IAAIc,SAAS/B,IAAI,CAACmB,UAAU3C,QAAQyC,cAAcI;IACvE,IAAIH,eAAe,IAAIa,SAAS/B,IAAI,CAACmB,UAAU1C,cAAcyC,YAAYG;IAEzE,8FAA8F;IAC9F,0FAA0F;IAC1F,4FAA4F;IAC5F,uFAAuF;IACvF,MAAMW,SAAS,CAAC;MACZ,EAAED,SAASlB,IAAI,CAAC,eAAe;4CACO,EAAEkB,SAAShC,MAAM,EAAE;IAC7D,MAAMkC,eAAezC,QAAQO,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,aAAa1C,QAAQO,MAAM,GAAG,IAAI,4CAA4C;IACpF,MAAMoC,MAAM,CAAC,wBAAwB,EAAEF,aAAa;IAClD,EAAED,OAAO;MACP,EAAEE,WAAW,gCAAgC,EAAE1C,QAAQO,MAAM,GAAG,IAAI,KAAK,YAAY;IACzF,MAAMqC,OAAO,MAAMP,KAAKQ,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAK/C,QAAQO,MAAM,GAAG,IAAI,EAAE,GAAG;QAACsB,KAAKmB,KAAK;KAAC;IACvE,+FAA+F;IAC/F,yFAAyF;IACzF,IAAIF,KAAK1B,IAAI,CAAC,CAAC6B,IAAMA,EAAEC,IAAI,KAAK,OAAO;QACrC,MAAM,IAAItE,WAAW,qBAAqB;IAC5C;IACA,MAAMuE,UAAUnD,QAAQO,MAAM,KAAK,IAAIuC,OAAOA,KAAKrC,MAAM,CAAC,CAAC2C,MAAQpD,QAAQqD,KAAK,CAAC,CAAChD,SAAWnB,gBAAgBkC,IAAI,CAAC,CAACkC;oBAA8BF;uBAApBvE,qBAAoBuE,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAI/C;;IACnK,OAAO8C,QAAQI,KAAK,CAAC,GAAG1B,KAAKmB,KAAK,EAAEtC,GAAG,CAAC,CAAC,EAAEwC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAChE"}

package/dist/esm/store/turso/native.d.ts

@@ -1,2 +1,6 @@
 export declare const TURSO_PACKAGE = "@tursodatabase/database";
+export declare const CONNECT_OPTS: {
+    readonly timeout: 30000;
+    readonly experimental: readonly ["index_method"];
+};
 export declare function tursoApi(): Promise<typeof import('@tursodatabase/database')>;

package/dist/esm/store/turso/native.js

@@ -5,6 +5,14 @@ const TURSO = {
     pkg: TURSO_PACKAGE,
     sizeHint: '~16MB'
 };
+// Every connect() in this store goes through these: the same 30s floor sqlite opens with, and the
+// FTS index method ensureSchema() needs. reclaimSpace adds 'vacuum'; nothing else may (PLAN 3.54).
+export const CONNECT_OPTS = {
+    timeout: 30000,
+    experimental: [
+        'index_method'
+    ]
+};
 let tursoApiPromise;
 // One accessor every consumer must go through, not each its own bare-specifier import: Node
 // keeps returning a pre-install "not found" miss to any new lookup of the same path afterward.

package/dist/esm/store/turso/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// One accessor every consumer must go through, not each its own bare-specifier import: Node\n// keeps returning a pre-install \"not found\" miss to any new lookup of the same path afterward.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","TURSO_PACKAGE","TURSO","store","pkg","sizeHint","tursoApiPromise","tursoApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,gBAAgB,0BAA0B;AAEvD,MAAMC,QAA0B;IAAEC,OAAO;IAASC,KAAKH;IAAeI,UAAU;AAAQ;AAExF,IAAIC;AAEJ,4FAA4F;AAC5F,+FAA+F;AAC/F,OAAO,SAASC;IACd,IAAI,CAACD,iBAAiBA,kBAAkBP,cAAwDG,OAAOF;IACvG,OAAOM;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\n// Every connect() in this store goes through these: the same 30s floor sqlite opens with, and the\n// FTS index method ensureSchema() needs. reclaimSpace adds 'vacuum'; nothing else may (PLAN 3.54).\nexport const CONNECT_OPTS = { timeout: 30_000, experimental: ['index_method'] } as const;\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// One accessor every consumer must go through, not each its own bare-specifier import: Node\n// keeps returning a pre-install \"not found\" miss to any new lookup of the same path afterward.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","TURSO_PACKAGE","TURSO","store","pkg","sizeHint","CONNECT_OPTS","timeout","experimental","tursoApiPromise","tursoApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,gBAAgB,0BAA0B;AAEvD,MAAMC,QAA0B;IAAEC,OAAO;IAASC,KAAKH;IAAeI,UAAU;AAAQ;AAExF,kGAAkG;AAClG,mGAAmG;AACnG,OAAO,MAAMC,eAAe;IAAEC,SAAS;IAAQC,cAAc;QAAC;KAAe;AAAC,EAAW;AAEzF,IAAIC;AAEJ,4FAA4F;AAC5F,+FAA+F;AAC/F,OAAO,SAASC;IACd,IAAI,CAACD,iBAAiBA,kBAAkBV,cAAwDG,OAAOF;IACvG,OAAOS;AACT"}

package/dist/esm/store/turso/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "5";
+export declare const SCHEMA_VERSION = "8";
 export type { OpenResult };
 export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/esm/store/turso/open.js

@@ -5,18 +5,18 @@ import { activeFeatures, FEATURES } from '../../features/index.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { checkpointWal, createConnection } from './connection.js';
-import { TURSO_PACKAGE, tursoApi } from './native.js';
+import { CONNECT_OPTS, TURSO_PACKAGE, tursoApi } from './native.js';
 import { CONTENT_FTS_DDL, tursoDialect } from './reconcile.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.turso.db';
 // Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves
 // separately. Covers the FTS indexes, the "_ngram" sidecar columns, and embeddings.vector's width.
-export const SCHEMA_VERSION = '5';
+export const SCHEMA_VERSION = '8';
 // This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
 // extra native state to thread, unlike duckdb's separate instance/connection pair.
 async function ensureSchema(_handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
-    await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);
+    await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_stem TEXT, summary_stem TEXT, text_stem TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);
     for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
@@ -51,12 +51,12 @@ async function connect(dbPath, _cfg) {
     }
     let db;
     try {
-        // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;
-        // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).
+        // `timeout` is connect-time only in this client; the derived value is set via runtime PRAGMA
+        // below. Options in native.ts, shared with reclaimSpace's own connection.
         db = await turso.connect(dbPath, {
-            timeout: 30000,
+            ...CONNECT_OPTS,
             experimental: [
-                'index_method'
+                ...CONNECT_OPTS.experimental
             ]
         });
     } catch (err) {

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","checkpointWal","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,aAAa,EAAEC,gBAAgB,QAAQ,kBAAkB;AAClE,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,eAAesB,MAAMC,MAAgB;IACnC,MAAMnB,cAAcmB;IACpB,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,6BAA6B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGiB,UAAU,CAACT,MAAQ,iEAAiEU,IAAI,CAACV,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAe0B,UAAUzB,GAAmB;IACjD,OAAOf,gBAAgBe,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { CONNECT_OPTS, TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '8';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_stem TEXT, summary_stem TEXT, text_stem TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // `timeout` is connect-time only in this client; the derived value is set via runtime PRAGMA\n    // below. Options in native.ts, shared with reclaimSpace's own connection.\n    db = await turso.connect(dbPath, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","checkpointWal","createConnection","CONNECT_OPTS","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,aAAa,EAAEC,gBAAgB,QAAQ,kBAAkB;AAClE,SAASC,YAAY,EAAEC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACpE,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,4MAA4M,CAAC;IAC9N,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWrB,eAAekB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAErB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMuB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMd,QAAQc,MAAM,sBAAuB,MAAM,MAAMb,QAAQa,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMX,QAAQc,MAAM,gBAAiB,MAAM,MAAMb,QAAQa,MAAM,YAAYpB,iBAAiBqB,KAAKjB;AACxG;AAEA,eAAeuB,MAAMC,MAAgB;IACnC,MAAMpB,cAAcoB;IACpB,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAejC,YAAY,MAAMiC;QACrC,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,wBAAwB,EAAES,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,6FAA6F;QAC7F,0EAA0E;QAC1EA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAE,GAAGtB,YAAY;YAAE4B,cAAc;mBAAI5B,aAAa4B,YAAY;aAAC;QAAC;IACnG,EAAE,OAAOH,KAAK;QACZ,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,6BAA6B,EAAE8B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMX,iBAAiB4B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAME,mBAA0C;IACrDC,UAAUxB;IACVyB,eAAexB;IACfyB,kBAAkB5B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGgB,UAAU,CAACR,MAAQ,iEAAiES,IAAI,CAACT,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAeyB,UAAUxB,GAAmB;IACjD,OAAOhB,gBAAgBgB,KAAKkB;AAC9B"}

package/dist/esm/store/turso/reconcile.d.ts

@@ -1,5 +1,5 @@
 import type { ReconcileDialect } from '../types.js';
-export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
 export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare const FTS_REBUILD_THRESHOLD = 250;
 export declare const tursoDialect: ReconcileDialect;

package/dist/esm/store/turso/reconcile.js

@@ -2,6 +2,7 @@ import { SenseError } from '../../errors.js';
 import { hasUnspacedRun } from '../../text/segment.js';
 import { quoteIdent, recordReconcileDuration } from '../shared.js';
 import { BEGIN_WRITE } from '../transaction.js';
+import { stemFolded } from './lexical-text.js';
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
 // store/reconcile.ts. content is a plain table with "_ngram" sidecars for lexical.ts.
 // The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
@@ -9,7 +10,7 @@ import { BEGIN_WRITE } from '../transaction.js';
 // The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around
 // a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.
 export const CONTENT_FTS_DDL = [
-    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,
+    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')`,
     `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`
 ];
 export const CONTENT_FTS_NAMES = [
@@ -28,7 +29,7 @@ const MAX_FRONTMATTER_COLUMNS = 2000;
 // 2026-08-30; the derivation and its bounds are pinned in this file's spec.
 export const FTS_REBUILD_THRESHOLD = 250;
 // No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
-const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;
+const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text, title_stem, summary_stem, text_stem, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
 function contentRow(doc) {
     const { title, summary, text } = doc.search;
     return [
@@ -36,6 +37,9 @@ function contentRow(doc) {
         title,
         summary,
         text,
+        stemFolded(title),
+        stemFolded(summary),
+        stemFolded(text),
         ngramSidecar(title),
         ngramSidecar(summary),
         ngramSidecar(text)

package/dist/esm/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","quoteIdent","recordReconcileDuration","BEGIN_WRITE","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","reconcileContent","conn","touched","docs","delta","churn","reparsed","length","vanished","bulk","files","name","exec","runBatch","map","p","ddl","addColumns","names","tursoDialect","beginMode","checkColumnLimit","count","recordDuration"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,UAAU,EAAEC,uBAAuB,QAAQ,eAAe;AACnE,SAASC,WAAW,QAAQ,oBAAoB;AAGhD,qFAAqF;AACrF,sFAAsF;AAEtF,+FAA+F;AAC/F,2FAA2F;AAC3F,iGAAiG;AACjG,uGAAuG;AACvG,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOP,eAAeO,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,4FAA4F;AAC5F,4EAA4E;AAC5E,OAAO,MAAMC,wBAAwB,IAAI;AAEzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,uHAAuH,CAAC;AAEpJ,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AAC5G;AAEA,eAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;IAC3G,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAE3D,6FAA6F;IAC7F,qDAAqD;IACrD,MAAME,OAAOL,MAAMM,KAAK,CAACH,MAAM,KAAK,KAAKF,QAAQb;IACjD,IAAIiB,MAAM,KAAK,MAAME,QAAQvB,kBAAmB,MAAMa,KAAKW,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIT,QAAQK,MAAM,GAAG,GACnB,MAAMN,KAAKY,QAAQ,CACjB,wCACAX,QAAQY,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIZ,KAAKI,MAAM,GAAG,GAAG,MAAMN,KAAKY,QAAQ,CAACpB,oBAAoBU,KAAKW,GAAG,CAACpB;IACtE,IAAIe,MAAM,KAAK,MAAMO,OAAO7B,gBAAiB,MAAMc,KAAKW,IAAI,CAACI;AAC/D;AAEA,yFAAyF;AACzF,eAAeC,WAAWhB,IAAgB,EAAEiB,KAAe;IACzD,KAAK,MAAMP,QAAQO,MAAO,MAAMjB,KAAKW,IAAI,CAAC,CAAC,mCAAmC,EAAE5B,WAAW2B,OAAO;AACpG;AAEA,OAAO,MAAMQ,eAAiC;IAC5CC,WAAW,IAAMlC;IACjBmC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQ/B,yBAAyB;YACnC,MAAM,IAAIT,WACR,gBACA,CAAC,uBAAuB,EAAEwC,MAAM,2DAA2D,EAAE/B,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA0B;IACAjB;IACAuB,gBAAgBtC;AAClB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_stem, summary_stem, text_stem, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, stemFolded(title), stemFolded(summary), stemFolded(text), ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","quoteIdent","recordReconcileDuration","BEGIN_WRITE","stemFolded","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","reconcileContent","conn","touched","docs","delta","churn","reparsed","length","vanished","bulk","files","name","exec","runBatch","map","p","ddl","addColumns","names","tursoDialect","beginMode","checkColumnLimit","count","recordDuration"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,UAAU,EAAEC,uBAAuB,QAAQ,eAAe;AACnE,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,qFAAqF;AACrF,sFAAsF;AAEtF,+FAA+F;AAC/F,2FAA2F;AAC3F,iGAAiG;AACjG,uGAAuG;AACvG,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,mKAAmK,CAAC;IACrK,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOR,eAAeQ,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,4FAA4F;AAC5F,4EAA4E;AAC5E,OAAO,MAAMC,wBAAwB,IAAI;AAEzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,qKAAqK,CAAC;AAElM,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMJ,WAAWU;QAAQV,WAAWW;QAAUX,WAAWI;QAAOD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AACtK;AAEA,eAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;IAC3G,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAE3D,6FAA6F;IAC7F,qDAAqD;IACrD,MAAME,OAAOL,MAAMM,KAAK,CAACH,MAAM,KAAK,KAAKF,QAAQb;IACjD,IAAIiB,MAAM,KAAK,MAAME,QAAQvB,kBAAmB,MAAMa,KAAKW,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIT,QAAQK,MAAM,GAAG,GACnB,MAAMN,KAAKY,QAAQ,CACjB,wCACAX,QAAQY,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIZ,KAAKI,MAAM,GAAG,GAAG,MAAMN,KAAKY,QAAQ,CAACpB,oBAAoBU,KAAKW,GAAG,CAACpB;IACtE,IAAIe,MAAM,KAAK,MAAMO,OAAO7B,gBAAiB,MAAMc,KAAKW,IAAI,CAACI;AAC/D;AAEA,yFAAyF;AACzF,eAAeC,WAAWhB,IAAgB,EAAEiB,KAAe;IACzD,KAAK,MAAMP,QAAQO,MAAO,MAAMjB,KAAKW,IAAI,CAAC,CAAC,mCAAmC,EAAE7B,WAAW4B,OAAO;AACpG;AAEA,OAAO,MAAMQ,eAAiC;IAC5CC,WAAW,IAAMnC;IACjBoC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQ/B,yBAAyB;YACnC,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAEyC,MAAM,2DAA2D,EAAE/B,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA0B;IACAjB;IACAuB,gBAAgBvC;AAClB,EAAE"}

package/dist/esm/store/turso/sql-functions.d.ts

@@ -0,0 +1 @@
+export declare function rewriteFunctions(sql: string): string;