sensemaking 0.18.2 → 0.18.3

package/dist/esm/store/turso/store.js

@@ -0,0 +1,93 @@
+import { SenseError } from '../../errors.js';
+import { getColumns } from '../shared.js';
+import { withTransaction } from '../transaction.js';
+import { hasVectorRow, pendingRows } from '../vectors.js';
+import { fieldStats } from './fieldStats.js';
+import { reconcile } from './reconcile.js';
+// Phase 1: the portable surface only -- frontmatter, links, sections, tags, rank, raw SQL,
+// reconcile, transactions. Neither lexical (Tantivy FTS) nor vectors (native vector_distance_cos
+// scans) is claimed yet -- phase 2 and phase 3 of the plan add them -- so lexical.query() and the
+// vector-math methods throw the named capability error rather than answer differently.
+// pending()/hasVector() are plain IS NULL/IS NOT NULL checks with no engine-specific math, so
+// they're wired through the same store-agnostic helpers duckdb's store.ts reuses: harmless ahead
+// of phase 3 (unreachable in practice, since openStore() already rejects a turso tree whose
+// config uses vectors) and correct with no further change once vectors land.
+export const CAPABILITIES = new Set();
+function missing(name) {
+    throw new SenseError('STORE_CAPABILITY_MISSING', `store "turso" does not implement "${name}" in this build; set "store" to "sqlite" or "duckdb" in this tree's config`);
+}
+// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth
+// (see transaction.ts) is tracked against the same object everywhere.
+export function createStore(db, conn, cfg, baseDir) {
+    return {
+        name: 'turso',
+        capabilities: CAPABILITIES,
+        async exec (sql) {
+            await conn.exec(sql);
+        },
+        async prepare (sql) {
+            return conn.prepare(sql);
+        },
+        async runBatch (sql, paramRows) {
+            await conn.runBatch(sql, paramRows);
+        },
+        async transaction (fn) {
+            return withTransaction(conn, fn);
+        },
+        async reconcile () {
+            return reconcile(conn, cfg, baseDir);
+        },
+        docs: {
+            async columns () {
+                return [
+                    ...await getColumns(conn)
+                ];
+            },
+            fieldStats: (columns, scopeWhere)=>fieldStats(conn, columns, scopeWhere)
+        },
+        lexical: {
+            async query () {
+                return missing('lexical');
+            }
+        },
+        vectors: {
+            pending: ()=>pendingRows(conn),
+            async writeVectors () {
+                return missing('vectors');
+            },
+            async candidates () {
+                return missing('vectors');
+            },
+            async similar () {
+                return missing('vectors');
+            },
+            hasVector: (path)=>hasVectorRow(conn, path)
+        },
+        async engineStatus () {
+            // Read back rather than recomputed: this is what open() actually set (3x the largest
+            // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.
+            // Turso's raw PRAGMA busy_timeout names its result column "busy_timeout" (spike-verified),
+            // not "timeout" like real SQLite's own quirky column name for this one pragma.
+            const row = await (await db.prepare('PRAGMA busy_timeout')).get();
+            return {
+                busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)`
+            };
+        },
+        raw: {
+            async prepare (sql) {
+                const stmt = await db.prepare(sql);
+                stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision
+                return {
+                    columns: ()=>stmt.columns(),
+                    // sense sql streams through the client's own async generator.
+                    iterate: async function*(...params) {
+                        yield* stmt.iterate(...params);
+                    }
+                };
+            }
+        },
+        async close () {
+            await db.close();
+        }
+    };
+}

package/dist/esm/store/turso/store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { reconcile } from './reconcile.ts';\n\n// Phase 1: the portable surface only -- frontmatter, links, sections, tags, rank, raw SQL,\n// reconcile, transactions. Neither lexical (Tantivy FTS) nor vectors (native vector_distance_cos\n// scans) is claimed yet -- phase 2 and phase 3 of the plan add them -- so lexical.query() and the\n// vector-math methods throw the named capability error rather than answer differently.\n// pending()/hasVector() are plain IS NULL/IS NOT NULL checks with no engine-specific math, so\n// they're wired through the same store-agnostic helpers duckdb's store.ts reuses: harmless ahead\n// of phase 3 (unreachable in practice, since openStore() already rejects a turso tree whose\n// config uses vectors) and correct with no further change once vectors land.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set();\n\nfunction missing(name: 'lexical' | 'vectors'): never {\n  throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement \"${name}\" in this build; set \"store\" to \"sqlite\" or \"duckdb\" in this tree's config`);\n}\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      async query() {\n        return missing('lexical');\n      },\n    },\n    vectors: {\n      pending: () => pendingRows(conn),\n      async writeVectors() {\n        return missing('vectors');\n      },\n      async candidates() {\n        return missing('vectors');\n      },\n      async similar() {\n        return missing('vectors');\n      },\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      // Turso's raw PRAGMA busy_timeout names its result column \"busy_timeout\" (spike-verified),\n      // not \"timeout\" like real SQLite's own quirky column name for this one pragma.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["SenseError","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","reconcile","CAPABILITIES","Set","missing","name","createStore","db","conn","cfg","baseDir","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","candidates","similar","hasVector","path","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AAEA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,SAAS,QAAQ,iBAAiB;AAE3C,2FAA2F;AAC3F,iGAAiG;AACjG,kGAAkG;AAClG,uFAAuF;AACvF,8FAA8F;AAC9F,iGAAiG;AACjG,4FAA4F;AAC5F,6EAA6E;AAC7E,OAAO,MAAMC,eAAwC,IAAIC,MAAM;AAE/D,SAASC,QAAQC,IAA2B;IAC1C,MAAM,IAAIV,WAAW,4BAA4B,CAAC,kCAAkC,EAAEU,KAAK,0EAA0E,CAAC;AACxK;AAEA,8FAA8F;AAC9F,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,OAAO;QACLL,MAAM;QACNM,cAAcT;QACd,MAAMU,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOrB,gBAAgBW,MAAMU;QAC/B;QACA,MAAMjB;YACJ,OAAOA,UAAUO,MAAMC,KAAKC;QAC9B;QACAS,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMxB,WAAWY;iBAAO;YACtC;YACAR,YAAY,CAACoB,SAASC,aAAerB,WAAWQ,MAAMY,SAASC;QACjE;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOnB,QAAQ;YACjB;QACF;QACAoB,SAAS;YACPC,SAAS,IAAM1B,YAAYS;YAC3B,MAAMkB;gBACJ,OAAOtB,QAAQ;YACjB;YACA,MAAMuB;gBACJ,OAAOvB,QAAQ;YACjB;YACA,MAAMwB;gBACJ,OAAOxB,QAAQ;YACjB;YACAyB,WAAW,CAACC,OAAShC,aAAaU,MAAMsB;QAC1C;QACA,MAAMC;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,2FAA2F;YAC3F,+EAA+E;YAC/E,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAMzB,GAAGO,OAAO,CAAC,sBAAqB,EAAGmB,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAMrB,SAAQD,GAAW;gBACvB,MAAMuB,OAAO,MAAM7B,GAAGO,OAAO,CAACD;gBAC9BuB,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLjB,SAAS,IAAMgB,KAAKhB,OAAO;oBAC3B,8DAA8D;oBAC9DkB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMjC,GAAGiC,KAAK;QAChB;IACF;AACF"}

package/dist/esm/store/types.d.ts

@@ -1,3 +1,4 @@
+import type { StoreName } from '../config/index.js';
 export type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';
 export interface RunResult {
     changes: number | bigint;
@@ -18,8 +19,14 @@ export interface Connection {
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
 }
+export interface FieldStat {
+    field: string;
+    coverage: number;
+    type: string;
+}
 export interface DocumentStore {
     columns(): Promise<string[]>;
+    fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;
 }
 export interface LexicalHit {
     path: string;
@@ -73,7 +80,7 @@ export interface SqlSession {
     prepare(sql: string): Promise<RawStatement>;
 }
 export interface Store {
-    readonly name: 'sqlite' | 'duckdb';
+    readonly name: StoreName;
     readonly capabilities: ReadonlySet<Capability>;
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
@@ -87,5 +94,6 @@ export interface Store {
     lexical: LexicalIndex;
     vectors: VectorStore;
     raw: SqlSession;
+    engineStatus(): Promise<Record<string, string>>;
     close(): Promise<void>;
 }

package/dist/esm/store/types.js

@@ -1,11 +1,4 @@
 // The backing-store interface: a minimal portable statement surface (exec/prepare, used as-is
 // by feature-owned tables and ad hoc queries) plus dedicated interfaces exactly where engines
 // diverge (lexical index, vector scan, raw sql passthrough).
-// 'lexical'/'vectors' mean the store's LexicalIndex/VectorStore are functionally implemented
-// (rather than present-but-inert); 'phrases'/'snippets'/'watch-concurrency' are the finer
-// behaviors sqlite's FTS5 path carries. A tree whose config needs a capability the chosen
-// store lacks fails at open (or at first use for a lazily-detected need), named.
-// 'phrases' means quoted-phrase (`"..."`) matching only -- not FTS5's wider query grammar
-// (prefix `*`, boolean AND/OR/NOT, NEAR, `^`, column filters), which duckdb's lexical path
-// rejects loudly rather than interpret (see store/duckdb/lexical.ts).
 export { };

package/dist/esm/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare, used as-is\n// by feature-owned tables and ad hoc queries) plus dedicated interfaces exactly where engines\n// diverge (lexical index, vector scan, raw sql passthrough).\n\n// 'lexical'/'vectors' mean the store's LexicalIndex/VectorStore are functionally implemented\n// (rather than present-but-inert); 'phrases'/'snippets'/'watch-concurrency' are the finer\n// behaviors sqlite's FTS5 path carries. A tree whose config needs a capability the chosen\n// store lacks fails at open (or at first use for a lazily-detected need), named.\n// 'phrases' means quoted-phrase (`\"...\"`) matching only -- not FTS5's wider query grammar\n// (prefix `*`, boolean AND/OR/NOT, NEAR, `^`, column filters), which duckdb's lexical path\n// rejects loudly rather than interpret (see store/duckdb/lexical.ts).\nexport type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every engine this store supports has to cross a real\n// async boundary for a query (DuckDB has no synchronous client at all), so run/get/all return\n// Promises. iterate() stays an async iterable for the same reason `sense sql` streams.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's own hot loops (schema,\n// reconcile). Portable so a feature never imports an engine-specific client type. `runBatch`\n// is the one call a write loop goes through instead of preparing once and calling `run()` per\n// row itself: one crossing per loop, the engine's own bulk idiom underneath (see each store's\n// implementation for what that idiom is).\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is still NULL (never embedded, or added since).\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: 'sqlite' | 'duckdb';\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, the graph\n  // ring's temp-table writes, etc.) goes through this instead of looping over `run()` itself.\n  // See Connection.runBatch -- same contract, same per-store implementation.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A command whose\n  // scope holds a network-bound write (search's embedding top-up) must stay outside it.\n  // Nesting joins the enclosing transaction rather than opening a second one.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  close(): Promise<void>;\n}\n"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,8FAA8F;AAC9F,6DAA6D;AAE7D,6FAA6F;AAC7F,0FAA0F;AAC1F,0FAA0F;AAC1F,iFAAiF;AACjF,0FAA0F;AAC1F,2FAA2F;AAC3F,sEAAsE;AA8FtE,WAqBC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare, used as-is\n// by feature-owned tables and ad hoc queries) plus dedicated interfaces exactly where engines\n// diverge (lexical index, vector scan, raw sql passthrough).\n\nimport type { StoreName } from '../config/index.ts';\n\n// 'lexical'/'vectors' mean the store's LexicalIndex/VectorStore are functionally implemented\n// (rather than present-but-inert); 'phrases'/'snippets'/'watch-concurrency' are the finer\n// behaviors sqlite's FTS5 path carries. A tree whose config needs a capability the chosen\n// store lacks fails at open (or at first use for a lazily-detected need), named.\n// 'phrases' means quoted-phrase (`\"...\"`) matching only -- not FTS5's wider query grammar\n// (prefix `*`, boolean AND/OR/NOT, NEAR, `^`, column filters), which duckdb's lexical path\n// rejects loudly rather than interpret (see store/duckdb/lexical.ts).\nexport type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every engine this store supports has to cross a real\n// async boundary for a query (DuckDB has no synchronous client at all), so run/get/all return\n// Promises. iterate() stays an async iterable for the same reason `sense sql` streams.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's own hot loops (schema,\n// reconcile). Portable so a feature never imports an engine-specific client type. `runBatch`\n// is the one call a write loop goes through instead of preparing once and calling `run()` per\n// row itself: one crossing per loop, the engine's own bulk idiom underneath (see each store's\n// implementation for what that idiom is).\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in SQL -- one query\n  // per call, never one row per note. `columns` is one chunk (caller owns the chunk size, a\n  // result-row column limit); `scopeWhere` is a caller-built WHERE fragment, same convention as\n  // LexicalQueryOptions. Each store expresses \"observed type\" through its own engine mechanism\n  // (native-not-emulated) but returns the shared integer/real/text vocabulary.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is still NULL (never embedded, or added since).\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, the graph\n  // ring's temp-table writes, etc.) goes through this instead of looping over `run()` itself.\n  // See Connection.runBatch -- same contract, same per-store implementation.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A command whose\n  // scope holds a network-bound write (search's embedding top-up) must stay outside it.\n  // Nesting joins the enclosing transaction rather than opening a second one.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading), each\n  // store owning what it reports and how it is worded; the command prints entries generically,\n  // one line per fact, without knowing any store's name. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,8FAA8F;AAC9F,6DAA6D;AAoH7D,WAyBC"}

package/dist/esm/store/vectors.d.ts

@@ -0,0 +1,9 @@
+import type { Connection } from './types.js';
+export declare const TARGET_CHUNK_CAP = 16;
+export declare function sampleEvenly<T>(rows: T[], cap?: number): T[];
+export declare function asCosine(score: number): number;
+export declare function pendingRows(conn: Connection): Promise<Array<{
+    path: string;
+    chunk: number;
+}>>;
+export declare function hasVectorRow(conn: Connection, path: string): Promise<boolean>;

package/dist/esm/store/vectors.js

@@ -0,0 +1,26 @@
+// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a
+// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).
+export const TARGET_CHUNK_CAP = 16;
+// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote
+// instead of being cut off by a fixed prefix.
+export function sampleEvenly(rows, cap = TARGET_CHUNK_CAP) {
+    const step = Math.max(1, Math.ceil(rows.length / cap));
+    return rows.filter((_, i)=>i % step === 0);
+}
+// Dequantised int8 dot products (sqlite) land a little either side of a true cosine, so an
+// identical pair can print 1.001; array_cosine_similarity (duckdb) has no such error but is
+// rounded the same way, so both stores print the same bounded number.
+export function asCosine(score) {
+    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
+}
+export async function pendingRows(conn) {
+    const stmt = await conn.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk');
+    return await stmt.all();
+}
+// Whether a note has any chunk with a vector. Distinguishes "nothing is near this note" from
+// "this note has no text", which look the same in an empty result.
+export async function hasVectorRow(conn, path) {
+    const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE "path" = ? AND vector IS NOT NULL LIMIT 1');
+    const row = await stmt.get(path);
+    return row !== undefined;
+}

package/dist/esm/store/vectors.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Dequantised int8 dot products (sqlite) land a little either side of a true cosine, so an\n// identical pair can print 1.001; array_cosine_similarity (duckdb) has no such error but is\n// rounded the same way, so both stores print the same bounded number.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","pendingRows","conn","stmt","prepare","all","hasVectorRow","path","row","get","undefined"],"mappings":"AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMA,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,sEAAsE;AACtE,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,OAAO,eAAeG,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEK,IAAY;IAC/D,MAAMJ,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMI,MAAO,MAAML,KAAKM,GAAG,CAACF;IAC5B,OAAOC,QAAQE;AACjB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.18.2",
+  "version": "0.18.3",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -97,6 +97,7 @@
   },
   "devDependencies": {
     "@duckdb/node-api": "*",
+    "@tursodatabase/database": "*",
     "@types/mdast": "^4.0.4",
     "@types/mocha": "*",
     "@types/node": "*",