sensemaking 0.25.0 → 0.26.0

package/dist/esm/store/turso/store.js

@@ -1,11 +1,10 @@
 import { STORE_DIMS } from '../../embed/types.js';
 import { getColumns, getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
-import { hasVectorRow, pendingRows } from '../vectors.js';
+import { createNativeVectorScopeOwner, hasVectorRow, pendingRows } from '../vectors.js';
 import { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
-import { rewriteFunctions } from './sql-functions.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 // No 'segment': has()/basename() resolve via SQL rewrite below (sql-functions.ts), but segment() has no SQL form and the client cannot register it as a UDF.
 export const CAPABILITIES = new Set([
@@ -17,6 +16,17 @@ const BLOAT_FACTOR = 1.5;
 // Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth
 // (see transaction.ts) is tracked against the same object everywhere.
 export function createStore(db, conn, options = {}) {
+    const vectorScope = createNativeVectorScopeOwner();
+    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the
+    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.
+    const vectors = {
+        pending: ()=>pendingRows(conn),
+        writeVectors: (rows)=>writeVectorBatch(conn, STORE_DIMS, rows),
+        candidates: (qv, _storeDims, fetch, allowed)=>scanCandidates(conn, qv, STORE_DIMS, fetch, allowed, vectorScope.activeFor(allowed)),
+        similar: (path, opts)=>scanSimilar(conn, STORE_DIMS, path, opts, vectorScope.activeFor(opts.allowed)),
+        hasVector: (path)=>hasVectorRow(conn, path)
+    };
+    vectorScope.bind(vectors);
     return {
         name: 'turso',
         capabilities: CAPABILITIES,
@@ -43,27 +53,19 @@ export function createStore(db, conn, options = {}) {
         lexical: {
             query: (terms, opts)=>queryLexical(conn, terms, opts)
         },
-        // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the
-        // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.
-        vectors: {
-            pending: ()=>pendingRows(conn),
-            writeVectors: (rows)=>writeVectorBatch(conn, STORE_DIMS, rows),
-            candidates: (qv, _storeDims, fetch, allowed)=>scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),
-            similar: (path, opts)=>scanSimilar(conn, STORE_DIMS, path, opts),
-            hasVector: (path)=>hasVectorRow(conn, path)
-        },
+        vectors,
         async engineStatus () {
             // Read back rather than recomputed: this is what open() actually set (3x the largest
             // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column "busy_timeout" (spike-verified), not "timeout" like real SQLite.
-            const row = await (await db.prepare('PRAGMA busy_timeout')).get();
+            const row = await (await conn.prepare('PRAGMA busy_timeout')).get();
             return {
                 busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)`
             };
         },
         raw: {
             async prepare (sql) {
-                const stmt = await db.prepare(rewriteFunctions(sql));
-                stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision
+                const stmt = await conn.prepare(sql);
+                stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of losing precision
                 return {
                     columns: ()=>stmt.columns(),
                     // sense sql streams through the client's own async generator.

package/dist/esm/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns, getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'segment': has()/basename() resolve via SQL rewrite below (sql-functions.ts), but segment() has no SQL form and the client cannot register it as a UDF.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors']);\n\n// close() reclaims once the cache file outgrows its compact size by this factor: the disk\n// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.\nconst BLOAT_FACTOR = 1.5;\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, options: { observational?: boolean } = {}): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(rewriteFunctions(sql));\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      if (options.observational) {\n        await db.close();\n        return;\n      }\n      // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.\n      await checkpointWal(db);\n      let reclaimPath: string | null = null;\n      try {\n        const path = await cacheFilePath(conn);\n        const base = Number((await getMeta(conn, 'compact_size')) ?? '0');\n        const size = path === null ? null : await fileSize(path);\n        if (size !== null) {\n          // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a\n          // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.\n          if (base === 0) {\n            await setMeta(conn, 'compact_size', String(size));\n            // The meta write landed past the checkpoint; leave no WAL for the next opener.\n            await checkpointWal(db);\n          } else if (size > BLOAT_FACTOR * base) {\n            reclaimPath = path;\n          }\n        }\n      } catch (err) {\n        console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${(err as Error).message}`);\n      }\n      await db.close();\n      // After this store's connection is closed, never before: turso hands the file to one\n      // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).\n      if (reclaimPath !== null) await reclaimSpace(reclaimPath);\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","getMeta","setMeta","withTransaction","hasVectorRow","pendingRows","cacheFilePath","checkpointWal","fileSize","reclaimSpace","fieldStats","queryLexical","rewriteFunctions","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","BLOAT_FACTOR","createStore","db","conn","options","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close","observational","reclaimPath","base","Number","size","String","err","console","error","message"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,EAAEC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAC5D,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,aAAa,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,YAAY,QAAQ,kBAAkB;AACvF,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,gBAAgB,QAAQ,qBAAqB;AACtD,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,6JAA6J;AAC7J,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;CAAU,EAAE;AAE1E,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,eAAe;AAErB,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB,EAAEC,UAAuC,CAAC,CAAC;IACnG,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO5B,gBAAgBkB,MAAMU;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMjC,WAAWqB;iBAAO;YACtC;YACAX,YAAY,CAACuB,SAASC,aAAexB,WAAWW,MAAMY,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAAS3B,aAAaU,MAAMgB,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAMnC,YAAYgB;YAC3BoB,cAAc,CAACC,OAAS3B,iBAAiBM,MAAMtB,YAAY2C;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYlC,eAAeQ,MAAMuB,IAAI7C,YAAY+C,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAASxB,YAAYO,MAAMtB,YAAYkD,MAAMX;YAC7DY,WAAW,CAACD,OAAS7C,aAAaiB,MAAM4B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAMhC,GAAGO,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMpC,GAAGO,OAAO,CAACf,iBAAiBc;gBAC/C8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,IAAItC,QAAQuC,aAAa,EAAE;gBACzB,MAAMzC,GAAGwC,KAAK;gBACd;YACF;YACA,4FAA4F;YAC5F,MAAMrD,cAAca;YACpB,IAAI0C,cAA6B;YACjC,IAAI;oBAEmB;gBADrB,MAAMb,OAAO,MAAM3C,cAAce;gBACjC,MAAM0C,OAAOC,QAAQ,OAAA,MAAM/D,QAAQoB,MAAM,6BAApB,kBAAA,OAAwC;gBAC7D,MAAM4C,OAAOhB,SAAS,OAAO,OAAO,MAAMzC,SAASyC;gBACnD,IAAIgB,SAAS,MAAM;oBACjB,yFAAyF;oBACzF,2FAA2F;oBAC3F,IAAIF,SAAS,GAAG;wBACd,MAAM7D,QAAQmB,MAAM,gBAAgB6C,OAAOD;wBAC3C,+EAA+E;wBAC/E,MAAM1D,cAAca;oBACtB,OAAO,IAAI6C,OAAO/C,eAAe6C,MAAM;wBACrCD,cAAcb;oBAChB;gBACF;YACF,EAAE,OAAOkB,KAAK;gBACZC,QAAQC,KAAK,CAAC,CAAC,yFAAyF,EAAE,AAACF,IAAcG,OAAO,EAAE;YACpI;YACA,MAAMlD,GAAGwC,KAAK;YACd,qFAAqF;YACrF,+FAA+F;YAC/F,IAAIE,gBAAgB,MAAM,MAAMrD,aAAaqD;QAC/C;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns, getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorStore } from '../types.ts';\nimport { createNativeVectorScopeOwner, hasVectorRow, pendingRows } from '../vectors.ts';\nimport { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'segment': has()/basename() resolve via SQL rewrite below (sql-functions.ts), but segment() has no SQL form and the client cannot register it as a UDF.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors']);\n\n// close() reclaims once the cache file outgrows its compact size by this factor: the disk\n// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.\nconst BLOAT_FACTOR = 1.5;\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, options: { observational?: boolean } = {}): Store {\n  const vectorScope = createNativeVectorScopeOwner();\n  // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n  // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n  const vectors: VectorStore = {\n    pending: () => pendingRows(conn),\n    writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n    candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed, vectorScope.activeFor(allowed)),\n    similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts, vectorScope.activeFor(opts.allowed)),\n    hasVector: (path) => hasVectorRow(conn, path),\n  };\n  vectorScope.bind(vectors);\n\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    vectors,\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await conn.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await conn.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      if (options.observational) {\n        await db.close();\n        return;\n      }\n      // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.\n      await checkpointWal(db);\n      let reclaimPath: string | null = null;\n      try {\n        const path = await cacheFilePath(conn);\n        const base = Number((await getMeta(conn, 'compact_size')) ?? '0');\n        const size = path === null ? null : await fileSize(path);\n        if (size !== null) {\n          // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a\n          // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.\n          if (base === 0) {\n            await setMeta(conn, 'compact_size', String(size));\n            // The meta write landed past the checkpoint; leave no WAL for the next opener.\n            await checkpointWal(db);\n          } else if (size > BLOAT_FACTOR * base) {\n            reclaimPath = path;\n          }\n        }\n      } catch (err) {\n        console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${(err as Error).message}`);\n      }\n      await db.close();\n      // After this store's connection is closed, never before: turso hands the file to one\n      // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).\n      if (reclaimPath !== null) await reclaimSpace(reclaimPath);\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","getMeta","setMeta","withTransaction","createNativeVectorScopeOwner","hasVectorRow","pendingRows","cacheFilePath","checkpointWal","fileSize","reclaimSpace","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","BLOAT_FACTOR","createStore","db","conn","options","vectorScope","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","activeFor","similar","path","opts","hasVector","bind","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","engineStatus","row","get","busy_timeout","raw","stmt","setReadBigInts","iterate","params","close","observational","reclaimPath","base","Number","size","String","err","console","error","message"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,EAAEC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAC5D,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,4BAA4B,EAAEC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AACxF,SAASC,aAAa,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,YAAY,QAAQ,kBAAkB;AACvF,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,6JAA6J;AAC7J,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;CAAU,EAAE;AAE1E,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,eAAe;AAErB,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB,EAAEC,UAAuC,CAAC,CAAC;IACnG,MAAMC,cAAcnB;IACpB,sFAAsF;IACtF,gIAAgI;IAChI,MAAMoB,UAAuB;QAC3BC,SAAS,IAAMnB,YAAYe;QAC3BK,cAAc,CAACC,OAASZ,iBAAiBM,MAAMtB,YAAY4B;QAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYnB,eAAeQ,MAAMQ,IAAI9B,YAAYgC,OAAOC,SAAST,YAAYU,SAAS,CAACD;QAC3HE,SAAS,CAACC,MAAMC,OAAStB,YAAYO,MAAMtB,YAAYoC,MAAMC,MAAMb,YAAYU,SAAS,CAACG,KAAKJ,OAAO;QACrGK,WAAW,CAACF,OAAS9B,aAAagB,MAAMc;IAC1C;IACAZ,YAAYe,IAAI,CAACd;IAEjB,OAAO;QACLe,MAAM;QACNC,cAAcxB;QACd,MAAMyB,MAAKC,GAAW;YACpB,MAAMrB,KAAKoB,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOrB,KAAKsB,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMxB,KAAKuB,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO5C,gBAAgBkB,MAAM0B;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMjD,WAAWqB;iBAAO;YACtC;YACAV,YAAY,CAACsC,SAASC,aAAevC,WAAWU,MAAM4B,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOjB,OAASxB,aAAaS,MAAMgC,OAAOjB;QACpD;QACAZ;QACA,MAAM8B;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAMlC,KAAKsB,OAAO,CAAC,sBAAqB,EAAGa,GAAG;YAClE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAMf,SAAQD,GAAW;gBACvB,MAAMiB,OAAO,MAAMtC,KAAKsB,OAAO,CAACD;gBAChCiB,KAAKC,cAAc,CAAC,OAAO,gEAAgE;gBAC3F,OAAO;oBACLX,SAAS,IAAMU,KAAKV,OAAO;oBAC3B,8DAA8D;oBAC9DY,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,IAAIzC,QAAQ0C,aAAa,EAAE;gBACzB,MAAM5C,GAAG2C,KAAK;gBACd;YACF;YACA,4FAA4F;YAC5F,MAAMvD,cAAcY;YACpB,IAAI6C,cAA6B;YACjC,IAAI;oBAEmB;gBADrB,MAAM9B,OAAO,MAAM5B,cAAcc;gBACjC,MAAM6C,OAAOC,QAAQ,OAAA,MAAMlE,QAAQoB,MAAM,6BAApB,kBAAA,OAAwC;gBAC7D,MAAM+C,OAAOjC,SAAS,OAAO,OAAO,MAAM1B,SAAS0B;gBACnD,IAAIiC,SAAS,MAAM;oBACjB,yFAAyF;oBACzF,2FAA2F;oBAC3F,IAAIF,SAAS,GAAG;wBACd,MAAMhE,QAAQmB,MAAM,gBAAgBgD,OAAOD;wBAC3C,+EAA+E;wBAC/E,MAAM5D,cAAcY;oBACtB,OAAO,IAAIgD,OAAOlD,eAAegD,MAAM;wBACrCD,cAAc9B;oBAChB;gBACF;YACF,EAAE,OAAOmC,KAAK;gBACZC,QAAQC,KAAK,CAAC,CAAC,yFAAyF,EAAE,AAACF,IAAcG,OAAO,EAAE;YACpI;YACA,MAAMrD,GAAG2C,KAAK;YACd,qFAAqF;YACrF,+FAA+F;YAC/F,IAAIE,gBAAgB,MAAM,MAAMvD,aAAauD;QAC/C;IACF;AACF"}

package/dist/esm/store/turso/vectors.d.ts

@@ -1,8 +1,8 @@
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
 export declare function writeVectorBatch(conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
-export declare function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export declare function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>, nativeScope?: boolean): Promise<VectorCandidate[]>;
 export declare function scanSimilar(conn: Connection, dims: number, path: string, opts: {
     exclude: Set<string>;
     allowed?: Set<string>;
     k: number;
-}): Promise<VectorSimilar[]>;
+}, nativeScope?: boolean): Promise<VectorSimilar[]>;

package/dist/esm/store/turso/vectors.js

@@ -1,4 +1,4 @@
-import { asCosine, compareVectorScores, sampleEvenly } from '../vectors.js';
+import { asCosine, compareVectorScores, NATIVE_VECTOR_SCOPE_TABLE, sampleEvenly } from '../vectors.js';
 // Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans
 // score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.
 // A model narrower than the column zero-pads: the added dimensions contribute nothing to
@@ -24,16 +24,15 @@ function dequantize(row, dims) {
     for(let d = 0; d < Math.min(q.length, dims); d++)out[d] = q[d] * row.scale;
     return out;
 }
-function cosineSimilarity(a, b) {
+function vectorNorm(vector) {
+    let norm = 0;
+    for(let d = 0; d < vector.length; d++)norm += vector[d] * vector[d];
+    return Math.sqrt(norm);
+}
+function cosineSimilarity(a, b, aNorm, bNorm) {
     let dot = 0;
-    let na = 0;
-    let nb = 0;
-    for(let d = 0; d < a.length; d++){
-        dot += a[d] * b[d];
-        na += a[d] * a[d];
-        nb += b[d] * b[d];
-    }
-    return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-32);
+    for(let d = 0; d < a.length; d++)dot += a[d] * b[d];
+    return dot / (aNorm * bNorm + 1e-32);
 }
 // `scale` stays NULL: the column exists so the row shape matches the other stores, and
 // cosine is scale-invariant once the value is dequantized here.
@@ -47,14 +46,17 @@ export async function writeVectorBatch(conn, dims, rows) {
 }
 // Best chunk per file by cosine. GROUP BY with MIN() cannot do this on 0.7.2: the aggregate
 // is right but the bare start_line/end_line come from an arbitrary row in the group.
-export async function scanCandidates(conn, qv, dims, fetch, allowed) {
+export async function scanCandidates(conn, qv, dims, fetch, allowed, nativeScope = false) {
     const q = padded(qv, dims);
-    const stmt = await conn.prepare('SELECT "path", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY "path", chunk');
+    const queryNorm = vectorNorm(q);
+    const scope = nativeScope ? ` AND "path" IN (SELECT "path" FROM ${NATIVE_VECTOR_SCOPE_TABLE})` : '';
+    const stmt = await conn.prepare(`SELECT "path", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL${scope} ORDER BY "path", chunk`);
     const rows = await stmt.all();
     const best = new Map();
     for (const row of rows){
         if (allowed && !allowed.has(row.path)) continue;
-        const score = cosineSimilarity(q, decode(row.vector, dims));
+        const vector = decode(row.vector, dims);
+        const score = cosineSimilarity(q, vector, queryNorm, vectorNorm(vector));
         const existing = best.get(row.path);
         if (!existing || score > existing.score || score === existing.score && row.chunk < existing.chunk) {
             best.set(row.path, {
@@ -77,20 +79,28 @@ export async function scanCandidates(conn, qv, dims, fetch, allowed) {
 }
 // Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured
 // 6-8x slower here at 26k chunks (2026-08-30).
-export async function scanSimilar(conn, dims, path, opts) {
+export async function scanSimilar(conn, dims, path, opts, nativeScope = false) {
     if (opts.allowed && opts.allowed.size === 0) return [];
     const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');
     const targetRows = await targetStmt.all(path);
     if (targetRows.length === 0) return [];
-    const targets = sampleEvenly(targetRows).map((row)=>decode(row.vector, dims));
-    const stmt = await conn.prepare('SELECT "path", vector FROM embeddings WHERE vector IS NOT NULL');
+    const targets = sampleEvenly(targetRows).map((row)=>{
+        const vector = decode(row.vector, dims);
+        return {
+            vector,
+            norm: vectorNorm(vector)
+        };
+    });
+    const scope = nativeScope ? ` AND "path" IN (SELECT "path" FROM ${NATIVE_VECTOR_SCOPE_TABLE})` : '';
+    const stmt = await conn.prepare(`SELECT "path", vector FROM embeddings WHERE vector IS NOT NULL${scope}`);
     const rows = await stmt.all();
     const best = new Map();
     for (const row of rows){
         if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
         const other = decode(row.vector, dims);
+        const otherNorm = vectorNorm(other);
         for (const t of targets){
-            const score = cosineSimilarity(t, other);
+            const score = cosineSimilarity(t.vector, other, t.norm, otherNorm);
             const existing = best.get(row.path);
             if (existing === undefined || score > existing) best.set(row.path, score);
         }

package/dist/esm/store/turso/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, compareVectorScores, sampleEvenly } from '../vectors.ts';\n\n// Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans\n// score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.\n\n// A model narrower than the column zero-pads: the added dimensions contribute nothing to\n// either dot product or norm, so cosine is unchanged.\nfunction padded(values: ArrayLike<number>, dims: number): Float32Array {\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\n// A bound Float32Array stores truncated bytes with no error; a Buffer view over the same\n// bytes round-trips exact (measured 0.7.2).\nfunction toBlob(v: Float32Array): Buffer {\n  return Buffer.from(v.buffer, v.byteOffset, v.byteLength);\n}\n\nfunction decode(vector: Buffer, dims: number): Float32Array {\n  return new Float32Array(vector.buffer, vector.byteOffset, dims);\n}\n\n// Rows arrive int8-quantized with a per-vector scale (embed/query.ts's toStore); this store\n// dequantizes to floats, as duckdb does.\nfunction dequantize(row: VectorWriteRow, dims: number): Float32Array {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(q.length, dims); d++) out[d] = q[d] * row.scale;\n  return out;\n}\n\nfunction cosineSimilarity(a: Float32Array, b: Float32Array): number {\n  let dot = 0;\n  let na = 0;\n  let nb = 0;\n  for (let d = 0; d < a.length; d++) {\n    dot += a[d] * b[d];\n    na += a[d] * a[d];\n    nb += b[d] * b[d];\n  }\n  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-32);\n}\n\n// `scale` stays NULL: the column exists so the row shape matches the other stores, and\n// cosine is scale-invariant once the value is dequantized here.\nexport async function writeVectorBatch(conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [toBlob(dequantize(row, dims)), row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine. GROUP BY with MIN() cannot do this on 0.7.2: the aggregate\n// is right but the bare start_line/end_line come from an arbitrary row in the group.\nexport async function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const q = padded(qv, dims);\n  const stmt = await conn.prepare('SELECT \"path\", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY \"path\", chunk');\n  const rows = (await stmt.all()) as Array<{ path: string; chunk: number; start_line: number; end_line: number; vector: Buffer }>;\n\n  const best = new Map<string, { score: number; chunk: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const score = cosineSimilarity(q, decode(row.vector, dims));\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score || (score === existing.score && row.chunk < existing.chunk)) {\n      best.set(row.path, { score, chunk: row.chunk, lines: `L${row.start_line}-${row.end_line}` });\n    }\n  }\n  return [...best.entries()]\n    .map(([path, value]) => ({ path, ...value }))\n    .sort(compareVectorScores)\n    .slice(0, fetch)\n    .map(({ path, lines, score }) => ({ path, lines, similarity: asCosine(score) }));\n}\n\n// Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured\n// 6-8x slower here at 26k chunks (2026-08-30).\nexport async function scanSimilar(conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Buffer }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => decode(row.vector, dims));\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Buffer }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = decode(row.vector, dims);\n    for (const t of targets) {\n      const score = cosineSimilarity(t, other);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .map(([p, score]) => ({ path: p, score }))\n    .sort(compareVectorScores)\n    .slice(0, opts.k)\n    .map(({ path: p, score }) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["asCosine","compareVectorScores","sampleEvenly","padded","values","dims","out","Float32Array","d","Math","min","length","toBlob","v","Buffer","from","buffer","byteOffset","byteLength","decode","vector","dequantize","row","q","Int8Array","scale","cosineSimilarity","a","b","dot","na","nb","sqrt","writeVectorBatch","conn","rows","runBatch","map","path","chunk","scanCandidates","qv","fetch","allowed","stmt","prepare","all","best","Map","has","score","existing","get","set","lines","start_line","end_line","entries","value","sort","slice","similarity","scanSimilar","opts","size","targetStmt","targetRows","targets","exclude","other","t","undefined","p","k"],"mappings":"AACA,SAASA,QAAQ,EAAEC,mBAAmB,EAAEC,YAAY,QAAQ,gBAAgB;AAE5E,0FAA0F;AAC1F,wFAAwF;AAExF,yFAAyF;AACzF,sDAAsD;AACtD,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOF;AACT;AAEA,yFAAyF;AACzF,4CAA4C;AAC5C,SAASM,OAAOC,CAAe;IAC7B,OAAOC,OAAOC,IAAI,CAACF,EAAEG,MAAM,EAAEH,EAAEI,UAAU,EAAEJ,EAAEK,UAAU;AACzD;AAEA,SAASC,OAAOC,MAAc,EAAEf,IAAY;IAC1C,OAAO,IAAIE,aAAaa,OAAOJ,MAAM,EAAEI,OAAOH,UAAU,EAAEZ;AAC5D;AAEA,4FAA4F;AAC5F,yCAAyC;AACzC,SAASgB,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,MAAMkB,IAAI,IAAIC,UAAUF,IAAIF,MAAM,CAACJ,MAAM,EAAEM,IAAIF,MAAM,CAACH,UAAU,EAAEK,IAAIF,MAAM,CAACF,UAAU;IACvF,MAAMZ,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACa,EAAEZ,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGe,CAAC,CAACf,EAAE,GAAGc,IAAIG,KAAK;IAC5E,OAAOnB;AACT;AAEA,SAASoB,iBAAiBC,CAAe,EAAEC,CAAe;IACxD,IAAIC,MAAM;IACV,IAAIC,KAAK;IACT,IAAIC,KAAK;IACT,IAAK,IAAIvB,IAAI,GAAGA,IAAImB,EAAEhB,MAAM,EAAEH,IAAK;QACjCqB,OAAOF,CAAC,CAACnB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;QAClBsB,MAAMH,CAAC,CAACnB,EAAE,GAAGmB,CAAC,CAACnB,EAAE;QACjBuB,MAAMH,CAAC,CAACpB,EAAE,GAAGoB,CAAC,CAACpB,EAAE;IACnB;IACA,OAAOqB,MAAOpB,CAAAA,KAAKuB,IAAI,CAACF,MAAMrB,KAAKuB,IAAI,CAACD,MAAM,KAAI;AACpD;AAEA,uFAAuF;AACvF,gEAAgE;AAChE,OAAO,eAAeE,iBAAiBC,IAAgB,EAAE7B,IAAY,EAAE8B,IAAsB;IAC3F,IAAIA,KAAKxB,MAAM,KAAK,GAAG;IACvB,MAAMuB,KAAKE,QAAQ,CACjB,mEACAD,KAAKE,GAAG,CAAC,CAACf,MAAQ;YAACV,OAAOS,WAAWC,KAAKjB;YAAQiB,IAAIgB,IAAI;YAAEhB,IAAIiB,KAAK;SAAC;AAE1E;AAEA,4FAA4F;AAC5F,qFAAqF;AACrF,OAAO,eAAeC,eAAeN,IAAgB,EAAEO,EAAgB,EAAEpC,IAAY,EAAEqC,KAAa,EAAEC,OAAqB;IACzH,MAAMpB,IAAIpB,OAAOsC,IAAIpC;IACrB,MAAMuC,OAAO,MAAMV,KAAKW,OAAO,CAAC;IAChC,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAE5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM1B,OAAOa,KAAM;QACtB,IAAIQ,WAAW,CAACA,QAAQM,GAAG,CAAC3B,IAAIgB,IAAI,GAAG;QACvC,MAAMY,QAAQxB,iBAAiBH,GAAGJ,OAAOG,IAAIF,MAAM,EAAEf;QACrD,MAAM8C,WAAWJ,KAAKK,GAAG,CAAC9B,IAAIgB,IAAI;QAClC,IAAI,CAACa,YAAYD,QAAQC,SAASD,KAAK,IAAKA,UAAUC,SAASD,KAAK,IAAI5B,IAAIiB,KAAK,GAAGY,SAASZ,KAAK,EAAG;YACnGQ,KAAKM,GAAG,CAAC/B,IAAIgB,IAAI,EAAE;gBAAEY;gBAAOX,OAAOjB,IAAIiB,KAAK;gBAAEe,OAAO,CAAC,CAAC,EAAEhC,IAAIiC,UAAU,CAAC,CAAC,EAAEjC,IAAIkC,QAAQ,EAAE;YAAC;QAC5F;IACF;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBpB,GAAG,CAAC,CAAC,CAACC,MAAMoB,MAAM,GAAM,CAAA;YAAEpB;YAAM,GAAGoB,KAAK;QAAC,CAAA,GACzCC,IAAI,CAAC1D,qBACL2D,KAAK,CAAC,GAAGlB,OACTL,GAAG,CAAC,CAAC,EAAEC,IAAI,EAAEgB,KAAK,EAAEJ,KAAK,EAAE,GAAM,CAAA;YAAEZ;YAAMgB;YAAOO,YAAY7D,SAASkD;QAAO,CAAA;AACjF;AAEA,6FAA6F;AAC7F,+CAA+C;AAC/C,OAAO,eAAeY,YAAY5B,IAAgB,EAAE7B,IAAY,EAAEiC,IAAY,EAAEyB,IAAgE;IAC9I,IAAIA,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACqB,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMC,aAAa,MAAM/B,KAAKW,OAAO,CAAC;IACtC,MAAMqB,aAAc,MAAMD,WAAWnB,GAAG,CAACR;IACzC,IAAI4B,WAAWvD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUjE,aAAagE,YAAY7B,GAAG,CAAC,CAACf,MAAQH,OAAOG,IAAIF,MAAM,EAAEf;IAEzE,MAAMuC,OAAO,MAAMV,KAAKW,OAAO,CAAC;IAChC,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAC5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM1B,OAAOa,KAAM;QACtB,IAAIb,IAAIgB,IAAI,KAAKA,QAAQyB,KAAKK,OAAO,CAACnB,GAAG,CAAC3B,IAAIgB,IAAI,KAAMyB,KAAKpB,OAAO,IAAI,CAACoB,KAAKpB,OAAO,CAACM,GAAG,CAAC3B,IAAIgB,IAAI,GAAI;QACtG,MAAM+B,QAAQlD,OAAOG,IAAIF,MAAM,EAAEf;QACjC,KAAK,MAAMiE,KAAKH,QAAS;YACvB,MAAMjB,QAAQxB,iBAAiB4C,GAAGD;YAClC,MAAMlB,WAAWJ,KAAKK,GAAG,CAAC9B,IAAIgB,IAAI;YAClC,IAAIa,aAAaoB,aAAarB,QAAQC,UAAUJ,KAAKM,GAAG,CAAC/B,IAAIgB,IAAI,EAAEY;QACrE;IACF;IAEA,OAAO;WAAIH,KAAKU,OAAO;KAAG,CACvBpB,GAAG,CAAC,CAAC,CAACmC,GAAGtB,MAAM,GAAM,CAAA;YAAEZ,MAAMkC;YAAGtB;QAAM,CAAA,GACtCS,IAAI,CAAC1D,qBACL2D,KAAK,CAAC,GAAGG,KAAKU,CAAC,EACfpC,GAAG,CAAC,CAAC,EAAEC,MAAMkC,CAAC,EAAEtB,KAAK,EAAE,GAAM,CAAA;YAAEZ,MAAMkC;YAAGX,YAAY7D,SAASkD;QAAO,CAAA;AACzE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, compareVectorScores, NATIVE_VECTOR_SCOPE_TABLE, sampleEvenly } from '../vectors.ts';\n\n// Native F32_BLOB(dims) columns, dims fixed at DDL time (open.ts, STORE_DIMS). Both scans\n// score in JS: a per-row vector_distance_cos measured 3.5-8x slower on this row engine.\n\n// A model narrower than the column zero-pads: the added dimensions contribute nothing to\n// either dot product or norm, so cosine is unchanged.\nfunction padded(values: ArrayLike<number>, dims: number): Float32Array {\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\n// A bound Float32Array stores truncated bytes with no error; a Buffer view over the same\n// bytes round-trips exact (measured 0.7.2).\nfunction toBlob(v: Float32Array): Buffer {\n  return Buffer.from(v.buffer, v.byteOffset, v.byteLength);\n}\n\nfunction decode(vector: Buffer, dims: number): Float32Array {\n  return new Float32Array(vector.buffer, vector.byteOffset, dims);\n}\n\n// Rows arrive int8-quantized with a per-vector scale (embed/query.ts's toStore); this store\n// dequantizes to floats, as duckdb does.\nfunction dequantize(row: VectorWriteRow, dims: number): Float32Array {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  const out = new Float32Array(dims);\n  for (let d = 0; d < Math.min(q.length, dims); d++) out[d] = q[d] * row.scale;\n  return out;\n}\n\nfunction vectorNorm(vector: Float32Array): number {\n  let norm = 0;\n  for (let d = 0; d < vector.length; d++) norm += vector[d] * vector[d];\n  return Math.sqrt(norm);\n}\n\nfunction cosineSimilarity(a: Float32Array, b: Float32Array, aNorm: number, bNorm: number): number {\n  let dot = 0;\n  for (let d = 0; d < a.length; d++) dot += a[d] * b[d];\n  return dot / (aNorm * bNorm + 1e-32);\n}\n\n// `scale` stays NULL: the column exists so the row shape matches the other stores, and\n// cosine is scale-invariant once the value is dequantized here.\nexport async function writeVectorBatch(conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [toBlob(dequantize(row, dims)), row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine. GROUP BY with MIN() cannot do this on 0.7.2: the aggregate\n// is right but the bare start_line/end_line come from an arbitrary row in the group.\nexport async function scanCandidates(conn: Connection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>, nativeScope = false): Promise<VectorCandidate[]> {\n  const q = padded(qv, dims);\n  const queryNorm = vectorNorm(q);\n  const scope = nativeScope ? ` AND \"path\" IN (SELECT \"path\" FROM ${NATIVE_VECTOR_SCOPE_TABLE})` : '';\n  const stmt = await conn.prepare(`SELECT \"path\", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL${scope} ORDER BY \"path\", chunk`);\n  const rows = (await stmt.all()) as Array<{ path: string; chunk: number; start_line: number; end_line: number; vector: Buffer }>;\n\n  const best = new Map<string, { score: number; chunk: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const vector = decode(row.vector, dims);\n    const score = cosineSimilarity(q, vector, queryNorm, vectorNorm(vector));\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score || (score === existing.score && row.chunk < existing.chunk)) {\n      best.set(row.path, { score, chunk: row.chunk, lines: `L${row.start_line}-${row.end_line}` });\n    }\n  }\n  return [...best.entries()]\n    .map(([path, value]) => ({ path, ...value }))\n    .sort(compareVectorScores)\n    .slice(0, fetch)\n    .map(({ path, lines, score }) => ({ path, lines, similarity: asCosine(score) }));\n}\n\n// Max cosine over (target chunk, other chunk) pairs. The SQL cross-join duckdb uses measured\n// 6-8x slower here at 26k chunks (2026-08-30).\nexport async function scanSimilar(conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }, nativeScope = false): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Buffer }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => {\n    const vector = decode(row.vector, dims);\n    return { vector, norm: vectorNorm(vector) };\n  });\n\n  const scope = nativeScope ? ` AND \"path\" IN (SELECT \"path\" FROM ${NATIVE_VECTOR_SCOPE_TABLE})` : '';\n  const stmt = await conn.prepare(`SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL${scope}`);\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Buffer }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = decode(row.vector, dims);\n    const otherNorm = vectorNorm(other);\n    for (const t of targets) {\n      const score = cosineSimilarity(t.vector, other, t.norm, otherNorm);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .map(([p, score]) => ({ path: p, score }))\n    .sort(compareVectorScores)\n    .slice(0, opts.k)\n    .map(({ path: p, score }) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["asCosine","compareVectorScores","NATIVE_VECTOR_SCOPE_TABLE","sampleEvenly","padded","values","dims","out","Float32Array","d","Math","min","length","toBlob","v","Buffer","from","buffer","byteOffset","byteLength","decode","vector","dequantize","row","q","Int8Array","scale","vectorNorm","norm","sqrt","cosineSimilarity","a","b","aNorm","bNorm","dot","writeVectorBatch","conn","rows","runBatch","map","path","chunk","scanCandidates","qv","fetch","allowed","nativeScope","queryNorm","scope","stmt","prepare","all","best","Map","has","score","existing","get","set","lines","start_line","end_line","entries","value","sort","slice","similarity","scanSimilar","opts","size","targetStmt","targetRows","targets","exclude","other","otherNorm","t","undefined","p","k"],"mappings":"AACA,SAASA,QAAQ,EAAEC,mBAAmB,EAAEC,yBAAyB,EAAEC,YAAY,QAAQ,gBAAgB;AAEvG,0FAA0F;AAC1F,wFAAwF;AAExF,yFAAyF;AACzF,sDAAsD;AACtD,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOF;AACT;AAEA,yFAAyF;AACzF,4CAA4C;AAC5C,SAASM,OAAOC,CAAe;IAC7B,OAAOC,OAAOC,IAAI,CAACF,EAAEG,MAAM,EAAEH,EAAEI,UAAU,EAAEJ,EAAEK,UAAU;AACzD;AAEA,SAASC,OAAOC,MAAc,EAAEf,IAAY;IAC1C,OAAO,IAAIE,aAAaa,OAAOJ,MAAM,EAAEI,OAAOH,UAAU,EAAEZ;AAC5D;AAEA,4FAA4F;AAC5F,yCAAyC;AACzC,SAASgB,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,MAAMkB,IAAI,IAAIC,UAAUF,IAAIF,MAAM,CAACJ,MAAM,EAAEM,IAAIF,MAAM,CAACH,UAAU,EAAEK,IAAIF,MAAM,CAACF,UAAU;IACvF,MAAMZ,MAAM,IAAIC,aAAaF;IAC7B,IAAK,IAAIG,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACa,EAAEZ,MAAM,EAAEN,OAAOG,IAAKF,GAAG,CAACE,EAAE,GAAGe,CAAC,CAACf,EAAE,GAAGc,IAAIG,KAAK;IAC5E,OAAOnB;AACT;AAEA,SAASoB,WAAWN,MAAoB;IACtC,IAAIO,OAAO;IACX,IAAK,IAAInB,IAAI,GAAGA,IAAIY,OAAOT,MAAM,EAAEH,IAAKmB,QAAQP,MAAM,CAACZ,EAAE,GAAGY,MAAM,CAACZ,EAAE;IACrE,OAAOC,KAAKmB,IAAI,CAACD;AACnB;AAEA,SAASE,iBAAiBC,CAAe,EAAEC,CAAe,EAAEC,KAAa,EAAEC,KAAa;IACtF,IAAIC,MAAM;IACV,IAAK,IAAI1B,IAAI,GAAGA,IAAIsB,EAAEnB,MAAM,EAAEH,IAAK0B,OAAOJ,CAAC,CAACtB,EAAE,GAAGuB,CAAC,CAACvB,EAAE;IACrD,OAAO0B,MAAOF,CAAAA,QAAQC,QAAQ,KAAI;AACpC;AAEA,uFAAuF;AACvF,gEAAgE;AAChE,OAAO,eAAeE,iBAAiBC,IAAgB,EAAE/B,IAAY,EAAEgC,IAAsB;IAC3F,IAAIA,KAAK1B,MAAM,KAAK,GAAG;IACvB,MAAMyB,KAAKE,QAAQ,CACjB,mEACAD,KAAKE,GAAG,CAAC,CAACjB,MAAQ;YAACV,OAAOS,WAAWC,KAAKjB;YAAQiB,IAAIkB,IAAI;YAAElB,IAAImB,KAAK;SAAC;AAE1E;AAEA,4FAA4F;AAC5F,qFAAqF;AACrF,OAAO,eAAeC,eAAeN,IAAgB,EAAEO,EAAgB,EAAEtC,IAAY,EAAEuC,KAAa,EAAEC,OAAqB,EAAEC,cAAc,KAAK;IAC9I,MAAMvB,IAAIpB,OAAOwC,IAAItC;IACrB,MAAM0C,YAAYrB,WAAWH;IAC7B,MAAMyB,QAAQF,cAAc,CAAC,mCAAmC,EAAE7C,0BAA0B,CAAC,CAAC,GAAG;IACjG,MAAMgD,OAAO,MAAMb,KAAKc,OAAO,CAAC,CAAC,2FAA2F,EAAEF,MAAM,uBAAuB,CAAC;IAC5J,MAAMX,OAAQ,MAAMY,KAAKE,GAAG;IAE5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM/B,OAAOe,KAAM;QACtB,IAAIQ,WAAW,CAACA,QAAQS,GAAG,CAAChC,IAAIkB,IAAI,GAAG;QACvC,MAAMpB,SAASD,OAAOG,IAAIF,MAAM,EAAEf;QAClC,MAAMkD,QAAQ1B,iBAAiBN,GAAGH,QAAQ2B,WAAWrB,WAAWN;QAChE,MAAMoC,WAAWJ,KAAKK,GAAG,CAACnC,IAAIkB,IAAI;QAClC,IAAI,CAACgB,YAAYD,QAAQC,SAASD,KAAK,IAAKA,UAAUC,SAASD,KAAK,IAAIjC,IAAImB,KAAK,GAAGe,SAASf,KAAK,EAAG;YACnGW,KAAKM,GAAG,CAACpC,IAAIkB,IAAI,EAAE;gBAAEe;gBAAOd,OAAOnB,IAAImB,KAAK;gBAAEkB,OAAO,CAAC,CAAC,EAAErC,IAAIsC,UAAU,CAAC,CAAC,EAAEtC,IAAIuC,QAAQ,EAAE;YAAC;QAC5F;IACF;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBvB,GAAG,CAAC,CAAC,CAACC,MAAMuB,MAAM,GAAM,CAAA;YAAEvB;YAAM,GAAGuB,KAAK;QAAC,CAAA,GACzCC,IAAI,CAAChE,qBACLiE,KAAK,CAAC,GAAGrB,OACTL,GAAG,CAAC,CAAC,EAAEC,IAAI,EAAEmB,KAAK,EAAEJ,KAAK,EAAE,GAAM,CAAA;YAAEf;YAAMmB;YAAOO,YAAYnE,SAASwD;QAAO,CAAA;AACjF;AAEA,6FAA6F;AAC7F,+CAA+C;AAC/C,OAAO,eAAeY,YAAY/B,IAAgB,EAAE/B,IAAY,EAAEmC,IAAY,EAAE4B,IAAgE,EAAEtB,cAAc,KAAK;IACnK,IAAIsB,KAAKvB,OAAO,IAAIuB,KAAKvB,OAAO,CAACwB,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMC,aAAa,MAAMlC,KAAKc,OAAO,CAAC;IACtC,MAAMqB,aAAc,MAAMD,WAAWnB,GAAG,CAACX;IACzC,IAAI+B,WAAW5D,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAM6D,UAAUtE,aAAaqE,YAAYhC,GAAG,CAAC,CAACjB;QAC5C,MAAMF,SAASD,OAAOG,IAAIF,MAAM,EAAEf;QAClC,OAAO;YAAEe;YAAQO,MAAMD,WAAWN;QAAQ;IAC5C;IAEA,MAAM4B,QAAQF,cAAc,CAAC,mCAAmC,EAAE7C,0BAA0B,CAAC,CAAC,GAAG;IACjG,MAAMgD,OAAO,MAAMb,KAAKc,OAAO,CAAC,CAAC,8DAA8D,EAAEF,OAAO;IACxG,MAAMX,OAAQ,MAAMY,KAAKE,GAAG;IAC5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAM/B,OAAOe,KAAM;QACtB,IAAIf,IAAIkB,IAAI,KAAKA,QAAQ4B,KAAKK,OAAO,CAACnB,GAAG,CAAChC,IAAIkB,IAAI,KAAM4B,KAAKvB,OAAO,IAAI,CAACuB,KAAKvB,OAAO,CAACS,GAAG,CAAChC,IAAIkB,IAAI,GAAI;QACtG,MAAMkC,QAAQvD,OAAOG,IAAIF,MAAM,EAAEf;QACjC,MAAMsE,YAAYjD,WAAWgD;QAC7B,KAAK,MAAME,KAAKJ,QAAS;YACvB,MAAMjB,QAAQ1B,iBAAiB+C,EAAExD,MAAM,EAAEsD,OAAOE,EAAEjD,IAAI,EAAEgD;YACxD,MAAMnB,WAAWJ,KAAKK,GAAG,CAACnC,IAAIkB,IAAI;YAClC,IAAIgB,aAAaqB,aAAatB,QAAQC,UAAUJ,KAAKM,GAAG,CAACpC,IAAIkB,IAAI,EAAEe;QACrE;IACF;IAEA,OAAO;WAAIH,KAAKU,OAAO;KAAG,CACvBvB,GAAG,CAAC,CAAC,CAACuC,GAAGvB,MAAM,GAAM,CAAA;YAAEf,MAAMsC;YAAGvB;QAAM,CAAA,GACtCS,IAAI,CAAChE,qBACLiE,KAAK,CAAC,GAAGG,KAAKW,CAAC,EACfxC,GAAG,CAAC,CAAC,EAAEC,MAAMsC,CAAC,EAAEvB,KAAK,EAAE,GAAM,CAAA;YAAEf,MAAMsC;YAAGZ,YAAYnE,SAASwD;QAAO,CAAA;AACzE"}

package/dist/esm/store/types.d.ts

@@ -27,6 +27,7 @@ export interface ReconcileDialect {
     beginMode(): string;
     checkColumnLimit(count: number): void;
     addColumns(conn: Connection, names: string[]): Promise<void>;
+    updateFrontmatter?(conn: Connection, writableColumns: string[], rows: unknown[][]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
 }
@@ -45,7 +46,9 @@ export interface OpenDialect<Handle> {
         observational?: boolean;
     }): Promise<void>;
     isLocked?(err: Error): boolean;
-    ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;
+    ensureCoreSchema(handle: Handle, conn: Connection): Promise<void>;
+    ensureFeatureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;
+    prepareConnection?(handle: Handle, conn: Connection, cfg: Config): Promise<void>;
     setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;
     prepareLexical?(conn: Connection): Promise<void>;
     assertLexicalReady?(conn: Connection): Promise<void>;

package/dist/esm/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'vectors': the store's VectorStore is functionally implemented, not present-but-inert; checked\n// at open (index.ts) before a config with an \"embed\" block is allowed onto a store lacking it.\n// 'segment': the engine can run segment() (Intl.Segmenter grapheme clustering) as a SQL function;\n// turso's client cannot register it and has no rewrite for it, unlike has()/basename(), which now\n// work everywhere. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['vectors', 'segment'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle, options?: { observational?: boolean }): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  // Persistent lexical preparation belongs to build, while no-build calls only the readiness\n  // check. SQLite and Turso maintain lexical state during reconcile and omit both hooks.\n  prepareLexical?(conn: Connection): Promise<void>;\n  assertLexicalReady?(conn: Connection): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig, options: { observational: boolean }): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\n// A ranked match, nothing more. Snippets are cut from the document above the store\n// (commands/search.ts), identically for every engine, so no store carries them.\nexport interface LexicalHit {\n  path: string;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with snippet, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,iGAAiG;AACjG,+FAA+F;AAC/F,kGAAkG;AAClG,kGAAkG;AAClG,qGAAqG;AACrG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;CAAU,CAAU"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'vectors': the store's VectorStore is functionally implemented, not present-but-inert; checked\n// at open (index.ts) before a config with an \"embed\" block is allowed onto a store lacking it.\n// 'segment': the engine can run segment() (Intl.Segmenter grapheme clustering) as a SQL function;\n// turso's client cannot register it and has no rewrite for it, unlike has()/basename(), which now\n// work everywhere. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['vectors', 'segment'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Replaces this engine's existing-row frontmatter upsert, inside the caller's\n  // publication transaction. Preserve non-writable columns and repair missing rows.\n  updateFrontmatter?(conn: Connection, writableColumns: string[], rows: unknown[][]): Promise<void>;\n  // Reconciles searchable content for `touched` paths and parsed `docs`; `delta` carries the tree\n  // state a strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle, options?: { observational?: boolean }): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Minimal, configuration-independent schema needed before the builder can capture a planning\n  // snapshot. This bootstrap may commit independently because no feature generation is published.\n  ensureCoreSchema(handle: Handle, conn: Connection): Promise<void>;\n  // Configuration-dependent persistent DDL. The builder invokes this only inside its fenced write\n  // transaction, so a failed feature enable cannot leave catalog changes under the old signature.\n  ensureFeatureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Connection-local preparation with no durable cache effect (DuckDB's TEMP vector stage).\n  prepareConnection?(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  // Persistent lexical preparation belongs to build, while no-build calls only the readiness\n  // check. SQLite and Turso maintain lexical state during reconcile and omit both hooks.\n  prepareLexical?(conn: Connection): Promise<void>;\n  assertLexicalReady?(conn: Connection): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig, options: { observational: boolean }): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\n// A ranked match, nothing more. Snippets are cut from the document above the store\n// (commands/search.ts), identically for every engine, so no store carries them.\nexport interface LexicalHit {\n  path: string;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with snippet, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,iGAAiG;AACjG,+FAA+F;AAC/F,kGAAkG;AAClG,kGAAkG;AAClG,qGAAqG;AACrG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;CAAU,CAAU"}

package/dist/esm/store/vectors.d.ts

@@ -1,4 +1,16 @@
-import type { Connection } from './types.js';
+import type { Connection, VectorStore } from './types.js';
+export declare const NATIVE_VECTOR_SCOPE_TABLE = "_vector_scope";
+interface NativeVectorScopeOwner {
+    bind(vectors: VectorStore): void;
+    activeFor(allowed: Set<string> | undefined): boolean;
+}
+interface NativeVectorScopeReservation {
+    activate(): void;
+    deactivate(): void;
+    invalidate(): void;
+}
+export declare function createNativeVectorScopeOwner(): NativeVectorScopeOwner;
+export declare function reserveNativeVectorScope(vectors: VectorStore, allowed: Set<string>): NativeVectorScopeReservation | null;
 export declare const TARGET_CHUNK_CAP = 16;
 export declare function sampleEvenly<T>(rows: T[], cap?: number): T[];
 export declare function asCosine(score: number): number;
@@ -14,3 +26,4 @@ export declare function pendingRows(conn: Connection): Promise<Array<{
     chunk: number;
 }>>;
 export declare function hasVectorRow(conn: Connection, path: string): Promise<boolean>;
+export {};

package/dist/esm/store/vectors.js

@@ -1,3 +1,58 @@
+export const NATIVE_VECTOR_SCOPE_TABLE = '_vector_scope';
+const nativeVectorScopeOwners = new WeakMap();
+// One owner is constructed beside one native store connection, then bound to that connection's
+// VectorStore wrapper. The adapter asks its own owner for a private boolean; no Set or reservation
+// can activate another Store, even when a caller reuses the same Set object across both.
+export function createNativeVectorScopeOwner() {
+    const owner = {};
+    return {
+        bind (vectors) {
+            if (owner.vectors) throw new Error('native vector scope owner is already bound');
+            if (nativeVectorScopeOwners.has(vectors)) throw new Error('VectorStore is already bound to a native vector scope owner');
+            owner.vectors = vectors;
+            nativeVectorScopeOwners.set(vectors, owner);
+        },
+        activeFor (allowed) {
+            var _owner_scope;
+            return allowed !== undefined && ((_owner_scope = owner.scope) === null || _owner_scope === void 0 ? void 0 : _owner_scope.phase) === 'active' && owner.scope.allowed === allowed;
+        }
+    };
+}
+export function reserveNativeVectorScope(vectors, allowed) {
+    const owner = nativeVectorScopeOwners.get(vectors);
+    if (!owner) return null;
+    if (owner.scope) throw new Error('native vector scope operation is already active for this Store');
+    const token = {};
+    owner.scope = {
+        token,
+        allowed,
+        phase: 'materializing'
+    };
+    let valid = true;
+    const current = ()=>{
+        var _owner_scope;
+        if (!valid || ((_owner_scope = owner.scope) === null || _owner_scope === void 0 ? void 0 : _owner_scope.token) !== token) throw new Error('native vector scope reservation is no longer valid');
+        return owner.scope;
+    };
+    return {
+        activate () {
+            const scope = current();
+            if (scope.phase !== 'materializing') throw new Error(`native vector scope cannot activate from ${scope.phase}`);
+            scope.phase = 'active';
+        },
+        deactivate () {
+            const scope = current();
+            if (scope.phase === 'cleaning') throw new Error('native vector scope cleanup already started');
+            scope.phase = 'cleaning';
+        },
+        invalidate () {
+            var _owner_scope;
+            if (!valid) return;
+            if (((_owner_scope = owner.scope) === null || _owner_scope === void 0 ? void 0 : _owner_scope.token) === token) owner.scope = undefined;
+            valid = false;
+        }
+    };
+}
 // Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a
 // heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).
 export const TARGET_CHUNK_CAP = 16;

package/dist/esm/store/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Every store computes a real cosine; clamp float error at the ends and round for the public score.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Rank by the computed cosine before rounding for display. Bytewise path order resolves only an\n// exact score tie, so two distinguishable similarities never cross a caller's result boundary.\nexport function compareVectorScores(a: { path: string; score: number }, b: { path: string; score: number }): number {\n  const byScore = b.score - a.score;\n  if (byScore !== 0) return byScore;\n  return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","compareVectorScores","a","b","byScore","Buffer","compare","from","path","pendingRows","conn","stmt","prepare","all","hasVectorRow","row","get","undefined"],"mappings":"AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMA,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,oGAAoG;AACpG,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,OAAO,SAASG,oBAAoBC,CAAkC,EAAEC,CAAkC;IACxG,MAAMC,UAAUD,EAAEL,KAAK,GAAGI,EAAEJ,KAAK;IACjC,IAAIM,YAAY,GAAG,OAAOA;IAC1B,OAAOC,OAAOC,OAAO,CAACD,OAAOE,IAAI,CAACL,EAAEM,IAAI,GAAGH,OAAOE,IAAI,CAACJ,EAAEK,IAAI;AAC/D;AAEA,OAAO,eAAeC,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEF,IAAY;IAC/D,MAAMG,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMG,MAAO,MAAMJ,KAAKK,GAAG,CAACR;IAC5B,OAAOO,QAAQE;AACjB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection, VectorStore } from './types.ts';\n\nexport const NATIVE_VECTOR_SCOPE_TABLE = '_vector_scope';\n\ntype NativeVectorScopePhase = 'materializing' | 'active' | 'cleaning';\n\ninterface NativeVectorScopeState {\n  token: object;\n  allowed: Set<string>;\n  phase: NativeVectorScopePhase;\n}\n\ninterface NativeVectorScopeOwnerState {\n  vectors?: VectorStore;\n  scope?: NativeVectorScopeState;\n}\n\ninterface NativeVectorScopeOwner {\n  bind(vectors: VectorStore): void;\n  activeFor(allowed: Set<string> | undefined): boolean;\n}\n\ninterface NativeVectorScopeReservation {\n  activate(): void;\n  deactivate(): void;\n  invalidate(): void;\n}\n\nconst nativeVectorScopeOwners = new WeakMap<VectorStore, NativeVectorScopeOwnerState>();\n\n// One owner is constructed beside one native store connection, then bound to that connection's\n// VectorStore wrapper. The adapter asks its own owner for a private boolean; no Set or reservation\n// can activate another Store, even when a caller reuses the same Set object across both.\nexport function createNativeVectorScopeOwner(): NativeVectorScopeOwner {\n  const owner: NativeVectorScopeOwnerState = {};\n  return {\n    bind(vectors) {\n      if (owner.vectors) throw new Error('native vector scope owner is already bound');\n      if (nativeVectorScopeOwners.has(vectors)) throw new Error('VectorStore is already bound to a native vector scope owner');\n      owner.vectors = vectors;\n      nativeVectorScopeOwners.set(vectors, owner);\n    },\n    activeFor(allowed) {\n      return allowed !== undefined && owner.scope?.phase === 'active' && owner.scope.allowed === allowed;\n    },\n  };\n}\n\nexport function reserveNativeVectorScope(vectors: VectorStore, allowed: Set<string>): NativeVectorScopeReservation | null {\n  const owner = nativeVectorScopeOwners.get(vectors);\n  if (!owner) return null;\n  if (owner.scope) throw new Error('native vector scope operation is already active for this Store');\n\n  const token = {};\n  owner.scope = { token, allowed, phase: 'materializing' };\n  let valid = true;\n\n  const current = (): NativeVectorScopeState => {\n    if (!valid || owner.scope?.token !== token) throw new Error('native vector scope reservation is no longer valid');\n    return owner.scope;\n  };\n\n  return {\n    activate() {\n      const scope = current();\n      if (scope.phase !== 'materializing') throw new Error(`native vector scope cannot activate from ${scope.phase}`);\n      scope.phase = 'active';\n    },\n    deactivate() {\n      const scope = current();\n      if (scope.phase === 'cleaning') throw new Error('native vector scope cleanup already started');\n      scope.phase = 'cleaning';\n    },\n    invalidate() {\n      if (!valid) return;\n      if (owner.scope?.token === token) owner.scope = undefined;\n      valid = false;\n    },\n  };\n}\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Every store computes a real cosine; clamp float error at the ends and round for the public score.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Rank by the computed cosine before rounding for display. Bytewise path order resolves only an\n// exact score tie, so two distinguishable similarities never cross a caller's result boundary.\nexport function compareVectorScores(a: { path: string; score: number }, b: { path: string; score: number }): number {\n  const byScore = b.score - a.score;\n  if (byScore !== 0) return byScore;\n  return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["NATIVE_VECTOR_SCOPE_TABLE","nativeVectorScopeOwners","WeakMap","createNativeVectorScopeOwner","owner","bind","vectors","Error","has","set","activeFor","allowed","undefined","scope","phase","reserveNativeVectorScope","get","token","valid","current","activate","deactivate","invalidate","TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","compareVectorScores","a","b","byScore","Buffer","compare","from","path","pendingRows","conn","stmt","prepare","all","hasVectorRow","row"],"mappings":"AAEA,OAAO,MAAMA,4BAA4B,gBAAgB;AA0BzD,MAAMC,0BAA0B,IAAIC;AAEpC,+FAA+F;AAC/F,mGAAmG;AACnG,yFAAyF;AACzF,OAAO,SAASC;IACd,MAAMC,QAAqC,CAAC;IAC5C,OAAO;QACLC,MAAKC,OAAO;YACV,IAAIF,MAAME,OAAO,EAAE,MAAM,IAAIC,MAAM;YACnC,IAAIN,wBAAwBO,GAAG,CAACF,UAAU,MAAM,IAAIC,MAAM;YAC1DH,MAAME,OAAO,GAAGA;YAChBL,wBAAwBQ,GAAG,CAACH,SAASF;QACvC;QACAM,WAAUC,OAAO;gBACiBP;YAAhC,OAAOO,YAAYC,aAAaR,EAAAA,eAAAA,MAAMS,KAAK,cAAXT,mCAAAA,aAAaU,KAAK,MAAK,YAAYV,MAAMS,KAAK,CAACF,OAAO,KAAKA;QAC7F;IACF;AACF;AAEA,OAAO,SAASI,yBAAyBT,OAAoB,EAAEK,OAAoB;IACjF,MAAMP,QAAQH,wBAAwBe,GAAG,CAACV;IAC1C,IAAI,CAACF,OAAO,OAAO;IACnB,IAAIA,MAAMS,KAAK,EAAE,MAAM,IAAIN,MAAM;IAEjC,MAAMU,QAAQ,CAAC;IACfb,MAAMS,KAAK,GAAG;QAAEI;QAAON;QAASG,OAAO;IAAgB;IACvD,IAAII,QAAQ;IAEZ,MAAMC,UAAU;YACAf;QAAd,IAAI,CAACc,SAASd,EAAAA,eAAAA,MAAMS,KAAK,cAAXT,mCAAAA,aAAaa,KAAK,MAAKA,OAAO,MAAM,IAAIV,MAAM;QAC5D,OAAOH,MAAMS,KAAK;IACpB;IAEA,OAAO;QACLO;YACE,MAAMP,QAAQM;YACd,IAAIN,MAAMC,KAAK,KAAK,iBAAiB,MAAM,IAAIP,MAAM,CAAC,yCAAyC,EAAEM,MAAMC,KAAK,EAAE;YAC9GD,MAAMC,KAAK,GAAG;QAChB;QACAO;YACE,MAAMR,QAAQM;YACd,IAAIN,MAAMC,KAAK,KAAK,YAAY,MAAM,IAAIP,MAAM;YAChDM,MAAMC,KAAK,GAAG;QAChB;QACAQ;gBAEMlB;YADJ,IAAI,CAACc,OAAO;YACZ,IAAId,EAAAA,eAAAA,MAAMS,KAAK,cAAXT,mCAAAA,aAAaa,KAAK,MAAKA,OAAOb,MAAMS,KAAK,GAAGD;YAChDM,QAAQ;QACV;IACF;AACF;AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMK,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,oGAAoG;AACpG,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,OAAO,SAASG,oBAAoBC,CAAkC,EAAEC,CAAkC;IACxG,MAAMC,UAAUD,EAAEL,KAAK,GAAGI,EAAEJ,KAAK;IACjC,IAAIM,YAAY,GAAG,OAAOA;IAC1B,OAAOC,OAAOC,OAAO,CAACD,OAAOE,IAAI,CAACL,EAAEM,IAAI,GAAGH,OAAOE,IAAI,CAACJ,EAAEK,IAAI;AAC/D;AAEA,OAAO,eAAeC,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEF,IAAY;IAC/D,MAAMG,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMG,MAAO,MAAMJ,KAAKjC,GAAG,CAAC8B;IAC5B,OAAOO,QAAQzC;AACjB"}

package/dist/esm/watch.d.ts

@@ -12,11 +12,17 @@ export type WatchEvent = {
     type: 'reconcile-error';
     message: string;
 };
+/** Options for the root {@link runWatch} API. */
 export interface WatchOptions {
     force?: boolean;
     onEvent?: (event: WatchEvent) => void;
+    /** Caller abort cancels provider work, skips final preparation, and resolves after cleanup. */
     signal?: AbortSignal;
     debounceMs?: number;
     heartbeatIntervalMs?: number;
 }
+/**
+ * Runs in the foreground. Caller abort waits for active native work and cleanup; SIGINT/SIGTERM
+ * also run one final freshness pass. Cleanup failures reject either shutdown path.
+ */
 export declare function runWatch(cfg: ResolvedConfig, opts?: WatchOptions): Promise<void>;