sensemaking 0.19.2 → 0.20.0

package/dist/esm/store/turso/reconcile.d.ts

@@ -1,3 +1,5 @@
 import type { ReconcileDialect } from '../types.js';
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare const FTS_REBUILD_THRESHOLD = 250;
 export declare const tursoDialect: ReconcileDialect;

package/dist/esm/store/turso/reconcile.js

@@ -2,9 +2,20 @@ import { SenseError } from '../../errors.js';
 import { hasUnspacedRun } from '../../text/segment.js';
 import { recordReconcileDuration } from '../shared.js';
 import { BEGIN_WRITE } from '../transaction.js';
-import { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.js';
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
 // store/reconcile.ts. content is a plain table with "_ngram" sidecars for lexical.ts.
+// The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
+// columns makes a bare substring match a whole word, defeating the prefix-query rejection.
+// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around
+// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.
+export const CONTENT_FTS_DDL = [
+    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,
+    `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`
+];
+export const CONTENT_FTS_NAMES = [
+    'content_fts',
+    'content_fts_ngram'
+];
 // '' when the field has no unspaced-script run (the common case), so the ngram index carries
 // nothing for it -- same "pay for nothing when absent" shape as sqlite's segmentField sidecars.
 function ngramSidecar(text) {

package/dist/esm/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","recordReconcileDuration","BEGIN_WRITE","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","reconcileContent","conn","touched","docs","delta","churn","reparsed","length","vanished","bulk","files","name","exec","runBatch","map","p","ddl","tursoDialect","beginMode","checkColumnLimit","count","columnTypeSuffix","recordDuration"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,uBAAuB,QAAQ,eAAe;AACvD,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,SAASC,eAAe,EAAEC,iBAAiB,QAAQ,YAAY;AAE/D,qFAAqF;AACrF,sFAAsF;AAEtF,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAON,eAAeM,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,4FAA4F;AAC5F,4EAA4E;AAC5E,OAAO,MAAMC,wBAAwB,IAAI;AAEzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,uHAAuH,CAAC;AAEpJ,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AAC5G;AAEA,eAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;IAC3G,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAE3D,6FAA6F;IAC7F,qDAAqD;IACrD,MAAME,OAAOL,MAAMM,KAAK,CAACH,MAAM,KAAK,KAAKF,QAAQb;IACjD,IAAIiB,MAAM,KAAK,MAAME,QAAQvB,kBAAmB,MAAMa,KAAKW,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIT,QAAQK,MAAM,GAAG,GACnB,MAAMN,KAAKY,QAAQ,CACjB,wCACAX,QAAQY,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIZ,KAAKI,MAAM,GAAG,GAAG,MAAMN,KAAKY,QAAQ,CAACpB,oBAAoBU,KAAKW,GAAG,CAACpB;IACtE,IAAIe,MAAM,KAAK,MAAMO,OAAO7B,gBAAiB,MAAMc,KAAKW,IAAI,CAACI;AAC/D;AAEA,OAAO,MAAMC,eAAiC;IAC5CC,WAAW,IAAMhC;IACjBiC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQ7B,yBAAyB;YACnC,MAAM,IAAIR,WACR,gBACA,CAAC,uBAAuB,EAAEqC,MAAM,2DAA2D,EAAE7B,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA8B,kBAAkB,IAAM;IACxBrB;IACAsB,gBAAgBrC;AAClB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","recordReconcileDuration","BEGIN_WRITE","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","reconcileContent","conn","touched","docs","delta","churn","reparsed","length","vanished","bulk","files","name","exec","runBatch","map","p","ddl","tursoDialect","beginMode","checkColumnLimit","count","columnTypeSuffix","recordDuration"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,uBAAuB,QAAQ,eAAe;AACvD,SAASC,WAAW,QAAQ,oBAAoB;AAGhD,qFAAqF;AACrF,sFAAsF;AAEtF,+FAA+F;AAC/F,2FAA2F;AAC3F,iGAAiG;AACjG,uGAAuG;AACvG,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAON,eAAeM,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,4FAA4F;AAC5F,4EAA4E;AAC5E,OAAO,MAAMC,wBAAwB,IAAI;AAEzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,uHAAuH,CAAC;AAEpJ,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AAC5G;AAEA,eAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;IAC3G,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAE3D,6FAA6F;IAC7F,qDAAqD;IACrD,MAAME,OAAOL,MAAMM,KAAK,CAACH,MAAM,KAAK,KAAKF,QAAQb;IACjD,IAAIiB,MAAM,KAAK,MAAME,QAAQvB,kBAAmB,MAAMa,KAAKW,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIT,QAAQK,MAAM,GAAG,GACnB,MAAMN,KAAKY,QAAQ,CACjB,wCACAX,QAAQY,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIZ,KAAKI,MAAM,GAAG,GAAG,MAAMN,KAAKY,QAAQ,CAACpB,oBAAoBU,KAAKW,GAAG,CAACpB;IACtE,IAAIe,MAAM,KAAK,MAAMO,OAAO7B,gBAAiB,MAAMc,KAAKW,IAAI,CAACI;AAC/D;AAEA,OAAO,MAAMC,eAAiC;IAC5CC,WAAW,IAAMhC;IACjBiC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQ7B,yBAAyB;YACnC,MAAM,IAAIR,WACR,gBACA,CAAC,uBAAuB,EAAEqC,MAAM,2DAA2D,EAAE7B,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA8B,kBAAkB,IAAM;IACxBrB;IACAsB,gBAAgBrC;AAClB,EAAE"}

package/dist/esm/store/turso/store.d.ts

@@ -1,5 +1,4 @@
 import type { Database } from '@tursodatabase/database';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: Database, conn: Connection): Store;

package/dist/esm/store/turso/store.js

@@ -1,11 +1,9 @@
 import { STORE_DIMS } from '../../embed/types.js';
-import { reconcile } from '../reconcile.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
-import { tursoDialect } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 // No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use
 // the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.
@@ -14,9 +12,9 @@ export const CAPABILITIES = new Set([
     'phrases',
     'vectors'
 ]);
-// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth
+// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth
 // (see transaction.ts) is tracked against the same object everywhere.
-export function createStore(db, conn, cfg, baseDir) {
+export function createStore(db, conn) {
     return {
         name: 'turso',
         capabilities: CAPABILITIES,
@@ -32,9 +30,6 @@ export function createStore(db, conn, cfg, baseDir) {
         async transaction (fn) {
             return withTransaction(conn, fn);
         },
-        async reconcile () {
-            return reconcile(conn, cfg, baseDir, tursoDialect);
-        },
         docs: {
             async columns () {
                 return [

package/dist/esm/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { tursoDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir, tursoDialect);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["STORE_DIMS","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","tursoDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,YAAY,QAAQ,iBAAiB;AAC9C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8FAA8F;AAC9F,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAM1B;YACJ,OAAOA,UAAUe,MAAMC,KAAKC,SAASV;QACvC;QACAoB,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACAV,YAAY,CAACuB,SAASC,aAAexB,WAAWU,MAAMa,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAAS3B,aAAaS,MAAMiB,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAM/B,YAAYW;YAC3BqB,cAAc,CAACC,OAAS3B,iBAAiBK,MAAMhB,YAAYsC;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYlC,eAAeO,MAAMwB,IAAIxC,YAAY0C,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAASxB,YAAYM,MAAMhB,YAAY6C,MAAMX;YAC7DY,WAAW,CAACD,OAASzC,aAAaY,MAAM6B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAMjC,GAAGQ,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMrC,GAAGQ,OAAO,CAACD;gBAC9B8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMzC,GAAGyC,KAAK;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACd,MAAMO,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOrB,gBAAgBY,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMxB,WAAWa;iBAAO;YACtC;YACAT,YAAY,CAACoB,SAASC,aAAerB,WAAWS,MAAMW,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAASxB,aAAaQ,MAAMe,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAM5B,YAAYU;YAC3BmB,cAAc,CAACC,OAASzB,iBAAiBK,MAAMd,YAAYkC;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,MAAMsB,IAAIpC,YAAYsC,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAAStB,YAAYM,MAAMd,YAAYyC,MAAMX;YAC7DY,WAAW,CAACD,OAAStC,aAAaW,MAAM2B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAM/B,GAAGM,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMnC,GAAGM,OAAO,CAACD;gBAC9B8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMvC,GAAGuC,KAAK;QAChB;IACF;AACF"}

package/dist/esm/store/types.d.ts

@@ -1,4 +1,4 @@
-import type { Config, StoreName } from '../config/index.js';
+import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
 export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "watch-concurrency", "lexical", "vectors", "sql-functions"];
@@ -29,6 +29,22 @@ export interface ReconcileDialect {
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
 }
+export interface OpenDialect<Handle> {
+    filename: string;
+    schemaVersion: string;
+    reconcileDialect: ReconcileDialect;
+    connect(dbPath: string, cfg: ResolvedConfig): Promise<{
+        handle: Handle;
+        conn: Connection;
+    }>;
+    close(handle: Handle): Promise<void>;
+    partialRebuild?(handle: Handle, conn: Connection, cfg: Config, changedKeys: Set<string>): Promise<boolean>;
+    postSchemaRebuild?(handle: Handle, conn: Connection, cfg: Config, baseDir: string, wantFeatures: string): Promise<string[]>;
+    extraRebuildReason?(handle: Handle): string | null;
+    ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;
+    setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;
+    createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;
+}
 export interface FieldStat {
     field: string;
     coverage: number;
@@ -96,10 +112,6 @@ export interface Store {
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
     transaction<T>(fn: () => Promise<T>): Promise<T>;
-    reconcile(): Promise<{
-        parsed: number;
-        warnings: string[];
-    }>;
     docs: DocumentStore;
     lexical: LexicalIndex;
     vectors: VectorStore;

package/dist/esm/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Type suffix for ALTER TABLE ADD COLUMN: '' for sqlite/turso, ' VARIANT' for duckdb.\n  columnTypeSuffix(): string;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,6FAA6F;AAC7F,mGAAmG;AACnG,mGAAmG;AACnG,kGAAkG;AAClG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,CAAU"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Type suffix for ALTER TABLE ADD COLUMN: '' for sqlite/turso, ' VARIANT' for duckdb.\n  columnTypeSuffix(): string;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db, tokenize}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // A signature change fully explained by this dialect's own carve-out (sqlite's tokenize-only\n  // partial rebuild): performs the DDL swap and returns true, or false when this change isn't one.\n  // Absent for duckdb/turso, which have no such carve-out.\n  partialRebuild?(handle: Handle, conn: Connection, cfg: Config, changedKeys: Set<string>): Promise<boolean>;\n  // Runs after ensureSchema when partialRebuild returned true this attempt: repopulates whatever\n  // the DDL swap emptied and records the adopted signature. sqlite-only.\n  postSchemaRebuild?(handle: Handle, conn: Connection, cfg: Config, baseDir: string, wantFeatures: string): Promise<string[]>;\n  // A rebuild trigger independent of the schema/feature signature (sqlite: the content table's own\n  // tokenizer no longer matches meta). Returns the rebuild reason, or null when none applies.\n  extraRebuildReason?(handle: Handle): string | null;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,6FAA6F;AAC7F,mGAAmG;AACnG,mGAAmG;AACnG,kGAAkG;AAClG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,CAAU"}

package/dist/esm/text/strip.js

@@ -1,10 +1,9 @@
-import { extractText, parse } from '../chunk/index.js';
+import { extractTexts, parse } from '../chunk/index.js';
 import { normalizeText } from '../scan/frontmatter.js';
-// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses
+// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses
 // (src/chunk) -- words and vectors read one definition of "the prose of this note".
 export function textFromBlocks(blocks) {
-    const text = blocks.map((block)=>extractText(block.node)).filter((s)=>s.length > 0).join('\n');
-    return normalizeText(text);
+    return normalizeText(extractTexts(blocks.map((block)=>block.node)));
 }
 // Thin parse + delegate, for callers with no blocks of their own already.
 export function stripText(value) {

package/dist/esm/text/strip.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractText, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  const text = blocks\n    .map((block) => extractText(block.node))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n  return normalizeText(text);\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractText","parse","normalizeText","textFromBlocks","blocks","text","map","block","node","filter","s","length","join","stripText","value"],"mappings":"AACA,SAASA,WAAW,EAAEC,KAAK,QAAQ,oBAAoB;AACvD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,4FAA4F;AAC5F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,MAAMC,OAAOD,OACVE,GAAG,CAAC,CAACC,QAAUP,YAAYO,MAAMC,IAAI,GACrCC,MAAM,CAAC,CAACC,IAAMA,EAAEC,MAAM,GAAG,GACzBC,IAAI,CAAC;IACR,OAAOV,cAAcG;AACvB;AAEA,0EAA0E;AAC1E,OAAO,SAASQ,UAAUC,KAAa;IACrC,OAAOX,eAAeF,MAAMa;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks.map((block) => block.node)));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractTexts","parse","normalizeText","textFromBlocks","blocks","map","block","node","stripText","value"],"mappings":"AACA,SAASA,YAAY,EAAEC,KAAK,QAAQ,oBAAoB;AACxD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,0FAA0F;AAC1F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,OAAOF,cAAcF,aAAaI,OAAOC,GAAG,CAAC,CAACC,QAAUA,MAAMC,IAAI;AACpE;AAEA,0EAA0E;AAC1E,OAAO,SAASC,UAAUC,KAAa;IACrC,OAAON,eAAeF,MAAMQ;AAC9B"}

package/dist/esm/watch.js

@@ -2,7 +2,7 @@ import { watch as fsWatch } from 'node:fs';
 import { STATE_DIR } from './config/index.js';
 import { SenseError } from './errors.js';
 import { guardedTick } from './lib/guarded-tick.js';
-import { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.js';
+import { docCount, getMeta, openStoreForWatch, requireWatchConcurrency, setMeta } from './store/index.js';
 // Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.
 const DEBOUNCE_MS = 200;
 const HEARTBEAT_INTERVAL_MS = 5000;
@@ -14,12 +14,13 @@ export async function runWatch(cfg, opts = {}) {
     const onEvent = (_opts_onEvent = opts.onEvent) !== null && _opts_onEvent !== void 0 ? _opts_onEvent : ()=>{};
     const debounceMs = (_opts_debounceMs = opts.debounceMs) !== null && _opts_debounceMs !== void 0 ? _opts_debounceMs : DEBOUNCE_MS;
     const heartbeatIntervalMs = (_opts_heartbeatIntervalMs = opts.heartbeatIntervalMs) !== null && _opts_heartbeatIntervalMs !== void 0 ? _opts_heartbeatIntervalMs : HEARTBEAT_INTERVAL_MS;
-    const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);
+    const { store, builder, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStoreForWatch(cfg);
     const baseDir = cfg.baseDir;
     const existingHeartbeat = await getMeta(store, 'watch_heartbeat');
     if (existingHeartbeat && !opts.force) {
         const age = Date.now() - Date.parse(existingHeartbeat);
         if (age >= 0 && age < STALE_HEARTBEAT_MS) {
+            await builder.close();
             await store.close();
             throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);
         }
@@ -53,7 +54,7 @@ export async function runWatch(cfg, opts = {}) {
             debounceTimer = null;
             inFlight = (async ()=>{
                 try {
-                    const { parsed, warnings } = await store.reconcile();
+                    const { parsed, warnings } = await builder.build();
                     onEvent({
                         type: 'reconciled',
                         parsed,
@@ -95,6 +96,7 @@ export async function runWatch(cfg, opts = {}) {
             if (debounceTimer) clearTimeout(debounceTimer);
             watcher.close();
             await inFlight;
+            await builder.close();
             await setMeta(store, 'watch_heartbeat', null);
             await setMeta(store, 'watch_pid', null);
             await store.close();

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","inFlight","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAElG,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUO;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMpB,QAAQa,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMhC,SAASc;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAIC,WAAiC;IACrC,MAAMC,oBAAoB;QACxB,IAAIF,eAAeG,aAAaH;QAChCA,gBAAgBI,WAAW;YACzBJ,gBAAgB;YAChBC,WAAW,AAAC,CAAA;gBACV,IAAI;oBACF,MAAM,EAAEtB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM8B,SAAS;oBAClDjC,QAAQ;wBAAEmB,MAAM;wBAAcZ;wBAAQc,OAAO,MAAMhC,SAASc;wBAAQE;oBAAS;gBAC/E,EAAE,OAAO6B,KAAK;oBACZlC,QAAQ;wBAAEmB,MAAM;wBAAmBgB,SAAS,AAACD,IAAcC,OAAO;oBAAC;gBACrE,SAAU;oBACRN,WAAW;gBACb;YACF,CAAA;QACF,GAAG5B;IACL;IAEA,iGAAiG;IACjG,uIAAuI;IACvI,MAAMmC,UAAUnD,QAAQwB,SAAS;QAAE4B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACtD,YAAY;QACpE4C;IACF;IACA,MAAMW,iBAAiBC,YACrBtD,YAAYmC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIyC,QAAc,CAACC;YAoBpB7C,cACCA;QApBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM8C,WAAW;gBAKf9C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQoB,GAAG,CAAC,UAAUD;YACtBnB,QAAQoB,GAAG,CAAC,WAAWD;aACvB9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaiD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIb,eAAeG,aAAaH;YAChCQ,QAAQpB,KAAK;YACb,MAAMa;YACN,MAAMpC,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB4B;QACF;QACAlB,QAAQwB,IAAI,CAAC,UAAUL;QACvBnB,QAAQwB,IAAI,CAAC,WAAWL;QACxB,KAAI9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaoD,OAAO,EAAEN;cACrB9C,gBAAAA,KAAKgD,MAAM,cAAXhD,oCAAAA,cAAaqD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStoreForWatch, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, builder, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStoreForWatch(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await builder.close();\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await builder.build();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await builder.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStoreForWatch","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","builder","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","inFlight","scheduleReconcile","clearTimeout","setTimeout","build","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,iBAAiB,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAE1G,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,OAAO,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMlB,kBAAkBO;IAC7G,MAAMY,UAAUZ,IAAIY,OAAO;IAE3B,MAAMC,oBAAoB,MAAMrB,QAAQa,OAAO;IAC/C,IAAIQ,qBAAqB,CAACZ,KAAKa,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMjB,oBAAoB;YACxC,MAAMQ,QAAQa,KAAK;YACnB,MAAMd,MAAMc,KAAK;YACjB,MAAM,IAAI9B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE+B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAb,QAAQ;QAAEoB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDT,QAAQ;YAAEoB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMjC,SAASc;YAAQG,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM/B,QAAQU,OAAO,mBAAmB,IAAIW,OAAOW,WAAW;QAC9D,MAAMhC,QAAQU,OAAO,aAAauB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAIC,WAAiC;IACrC,MAAMC,oBAAoB;QACxB,IAAIF,eAAeG,aAAaH;QAChCA,gBAAgBI,WAAW;YACzBJ,gBAAgB;YAChBC,WAAW,AAAC,CAAA;gBACV,IAAI;oBACF,MAAM,EAAEtB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,QAAQ8B,KAAK;oBAChDlC,QAAQ;wBAAEoB,MAAM;wBAAcZ;wBAAQc,OAAO,MAAMjC,SAASc;wBAAQG;oBAAS;gBAC/E,EAAE,OAAO6B,KAAK;oBACZnC,QAAQ;wBAAEoB,MAAM;wBAAmBgB,SAAS,AAACD,IAAcC,OAAO;oBAAC;gBACrE,SAAU;oBACRN,WAAW;gBACb;YACF,CAAA;QACF,GAAG7B;IACL;IAEA,iGAAiG;IACjG,uIAAuI;IACvI,MAAMoC,UAAUpD,QAAQyB,SAAS;QAAE4B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACvD,YAAY;QACpE6C;IACF;IACA,MAAMW,iBAAiBC,YACrBvD,YAAYoC,gBAAgB,IAAMD,WAClCrB;IAGF,OAAO,IAAI0C,QAAc,CAACC;YAqBpB9C,cACCA;QArBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM+C,WAAW;gBAKf/C;YAJA,IAAIwB,UAAU;YACdA,WAAW;YACXI,QAAQoB,GAAG,CAAC,UAAUD;YACtBnB,QAAQoB,GAAG,CAAC,WAAWD;aACvB/C,eAAAA,KAAKiD,MAAM,cAAXjD,mCAAAA,aAAakD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIb,eAAeG,aAAaH;YAChCQ,QAAQpB,KAAK;YACb,MAAMa;YACN,MAAM1B,QAAQa,KAAK;YACnB,MAAMxB,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMc,KAAK;YACjB4B;QACF;QACAlB,QAAQwB,IAAI,CAAC,UAAUL;QACvBnB,QAAQwB,IAAI,CAAC,WAAWL;QACxB,KAAI/C,eAAAA,KAAKiD,MAAM,cAAXjD,mCAAAA,aAAaqD,OAAO,EAAEN;cACrB/C,gBAAAA,KAAKiD,MAAM,cAAXjD,oCAAAA,cAAasD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.19.2",
+  "version": "0.20.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -82,7 +82,7 @@
   "dependencies": {
     "@huggingface/tokenizers": "^0.1.3",
     "franc-min": "^6.2.0",
-    "install-module-linked": "^1.6.5",
+    "install-module-linked": "^2.0.0",
     "mdast-util-from-markdown": "^2.0.3",
     "mdast-util-gfm-autolink-literal": "^2.0.0",
     "mdast-util-gfm-footnote": "^2.0.0",