sensemaking 0.19.2 → 0.20.0

package/dist/esm/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir, duckdbDialect);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","duckdbDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB,EAAE;AAEjH,8JAA8J;AAC9J,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAM7B;YACJ,MAAM8B,SAAS,MAAM9B,UAAUiB,MAAMC,KAAKC,SAASX;YACnD,uFAAuF;YACvF,4DAA4D;YAC5DY,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMlB,YAAYyC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI3C,YAAY6C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMlB,YAAYgD,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB,EAAE;AAEjH,mKAAmK;AACnK,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,MAAMC,MAAMV,mBAAmBS;IAC/B,OAAO;QACLE,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBc,MAAMU;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWe;iBAAO;YACtC;YACAV,YAAY,CAACsB,SAASC,aAAevB,WAAWU,MAAMY,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOd,IAAIc,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAM7B,YAAYY;YAC3BkB,cAAc,CAACC,OAASzB,iBAAiBK,QAAQC,MAAMhB,YAAYmC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,QAAQsB,IAAIrC,YAAYuC,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAASlC,YAAYM,QAAQC,MAAMhB,YAAY0C,MAAMC;YACrEC,WAAW,CAACF,OAASvC,aAAaa,MAAM0B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAMxB,SAAQD,GAAW;gBACvB,MAAM0B,WAAW,MAAMhC,OAAOO,OAAO,CAACD;gBACtC,OAAO;oBACLO,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEjC,MAAM6B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAMtD,aAAa8B,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFjD,OAAOkD,cAAc;YACrBnD,SAASoD,SAAS;QACpB;IACF;AACF"}

package/dist/esm/store/index.d.ts

@@ -1,10 +1,15 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { OpenResult } from './sqlite/open.js';
+import type { Builder } from './builder.js';
+import type { OpenResult } from './open.js';
 export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & {
+    builder: Builder;
+}>;
 export declare function requireWatchConcurrency(cfg: ResolvedConfig): void;
 export { clearCache } from './cache.js';
+export type { OpenResult } from './open.js';
+export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
-export type { OpenResult } from './sqlite/open.js';
-export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
 export { CAPABILITY_NAMES } from './types.js';

package/dist/esm/store/index.js

@@ -2,26 +2,30 @@
 // beyond frontmatter + content lives in src/features/.
 import { anyPresetEmbeds, storeName } from '../config/index.js';
 import { SenseError } from '../errors.js';
-import { openDuckdb } from './duckdb/open.js';
+import { duckdbOpenDialect, openDuckdb } from './duckdb/open.js';
 import { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.js';
-import { openSqlite } from './sqlite/open.js';
+import { openWithBuilder } from './open.js';
+import { openSqlite, sqliteOpenDialect } from './sqlite/open.js';
 import { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.js';
-import { openTurso } from './turso/open.js';
+import { openTurso, tursoOpenDialect } from './turso/open.js';
 import { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.js';
 // Capabilities are checked against the registry entry before opening (no wasted connect/reconcile
 // work for a config that cannot be satisfied), then the store itself opens.
 const REGISTRY = {
     sqlite: {
         capabilities: SQLITE_CAPABILITIES,
-        open: openSqlite
+        open: openSqlite,
+        openForWatch: (cfg)=>openWithBuilder(cfg, sqliteOpenDialect)
     },
     duckdb: {
         capabilities: DUCKDB_CAPABILITIES,
-        open: openDuckdb
+        open: openDuckdb,
+        openForWatch: (cfg)=>openWithBuilder(cfg, duckdbOpenDialect)
     },
     turso: {
         capabilities: TURSO_CAPABILITIES,
-        open: openTurso
+        open: openTurso,
+        openForWatch: (cfg)=>openWithBuilder(cfg, tursoOpenDialect)
     }
 };
 function entryFor(name) {
@@ -39,6 +43,16 @@ export async function openStore(cfg) {
     }
     return entry.open(cfg);
 }
+// Same capability-checked open as openStore, but for a caller (only src/watch.ts) that must
+// reconcile again later: keeps the builder's pool warm and hands it back instead of closing it.
+export async function openStoreForWatch(cfg) {
+    const name = storeName(cfg);
+    const entry = entryFor(name);
+    if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
+        throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "vectors" in this build, but this config's "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)`);
+    }
+    return entry.openForWatch(cfg);
+}
 // watch keeps its connection open for the run, so a file-locking store would fail every
 // other command on the tree; check before open, as openStore does for "vectors".
 export function requireWatchConcurrency(cfg) {
@@ -48,6 +62,7 @@ export function requireWatchConcurrency(cfg) {
     }
 }
 export { clearCache } from './cache.js';
+export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
-export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export { CAPABILITY_NAMES } from './types.js';

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","clearCache","getMeta","setMeta","DB_FILENAME","docCount","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AACxC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,SAASC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB;AAEzE,SAASC,gBAAgB,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Builder } from './builder.ts';\nimport { duckdbOpenDialect, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './open.ts';\nimport { openWithBuilder } from './open.ts';\nimport { openSqlite, sqliteOpenDialect } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso, tursoOpenDialect } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n  // Same open, but keeps the builder's parse pool alive and hands it back: only a watcher, which\n  // reconciles repeatedly on the same connection, needs this.\n  openForWatch: (cfg: ResolvedConfig) => Promise<OpenResult & { builder: Builder }>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite, openForWatch: (cfg) => openWithBuilder(cfg, sqliteOpenDialect) },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb, openForWatch: (cfg) => openWithBuilder(cfg, duckdbOpenDialect) },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso, openForWatch: (cfg) => openWithBuilder(cfg, tursoOpenDialect) },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// Same capability-checked open as openStore, but for a caller (only src/watch.ts) that must\n// reconcile again later: keeps the builder's pool warm and hands it back instead of closing it.\nexport async function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & { builder: Builder }> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.openForWatch(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport type { OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","duckdbOpenDialect","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openWithBuilder","openSqlite","sqliteOpenDialect","SQLITE_CAPABILITIES","openTurso","tursoOpenDialect","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","openForWatch","cfg","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","has","openStoreForWatch","requireWatchConcurrency","clearCache","docCount","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,iBAAiB,EAAEC,UAAU,QAAQ,mBAAmB;AACjE,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,eAAe,QAAQ,YAAY;AAC5C,SAASC,UAAU,EAAEC,iBAAiB,QAAQ,mBAAmB;AACjE,SAASJ,gBAAgBK,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,kBAAkB;AAC9D,SAASP,gBAAgBQ,kBAAkB,QAAQ,mBAAmB;AAWtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcN;QAAqBO,MAAMT;QAAYU,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKV;IAAmB;IAC9HW,QAAQ;QAAEJ,cAAcV;QAAqBW,MAAMb;QAAYc,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKhB;IAAmB;IAC9HkB,OAAO;QAAEL,cAAcH;QAAoBI,MAAMN;QAAWO,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKP;IAAkB;AAC5H;AAEA,SAASU,SAASC,IAAe;IAC/B,MAAMC,QAAQV,QAAQ,CAACS,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAItB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEqB,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACZ,UAAUa,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUT,GAAmB;IACjD,MAAMI,OAAOtB,UAAUkB;IACvB,MAAMK,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIvB,gBAAgBmB,QAAQ,CAACK,MAAMR,YAAY,CAACa,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMP,IAAI,CAACE;AACpB;AAEA,4FAA4F;AAC5F,gGAAgG;AAChG,OAAO,eAAeW,kBAAkBX,GAAmB;IACzD,MAAMI,OAAOtB,UAAUkB;IACvB,MAAMK,QAAQF,SAASC;IAEvB,IAAIvB,gBAAgBmB,QAAQ,CAACK,MAAMR,YAAY,CAACa,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMN,YAAY,CAACC;AAC5B;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASY,wBAAwBZ,GAAmB;IACzD,MAAMI,OAAOtB,UAAUkB;IACvB,IAAI,CAACG,SAASC,MAAMP,YAAY,CAACa,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AAExC,SAASC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,WAAW,EAAEC,cAAc,QAAQ,mBAAmB;AAE/D,SAASC,gBAAgB,QAAQ,aAAa"}

package/dist/esm/store/open.d.ts

@@ -0,0 +1,15 @@
+import type { ResolvedConfig } from '../config/index.js';
+import type { Builder } from './builder.js';
+import type { OpenDialect, Store } from './types.js';
+export interface OpenResult {
+    store: Store;
+    cfg: ResolvedConfig;
+    dbPath: string;
+    parsed: number;
+    warnings: string[];
+}
+export declare function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult>;
+export declare function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & {
+    builder: Builder;
+}>;
+export declare function docCount(store: Store): Promise<number>;

package/dist/esm/store/open.js

@@ -0,0 +1,137 @@
+import { mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { featureSignature, STATE_DIR } from '../config/index.js';
+import { rekeyChunkText } from '../embed/handoff.js';
+import { FEATURES } from '../features/index.js';
+import { createBuilder } from './builder.js';
+import { clearCache } from './cache.js';
+import { getMeta, setMeta } from './shared.js';
+import { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.js';
+async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
+    const stateDir = join(cfg.baseDir, STATE_DIR);
+    mkdirSync(stateDir, {
+        recursive: true
+    });
+    const dbPath = join(stateDir, dialect.filename);
+    const { handle, conn } = await dialect.connect(dbPath, cfg);
+    // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on
+    // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.
+    let closed = false;
+    // Created here, not inside reconcile(): its pool must survive across every build() call this
+    // connection sees (a watcher's repeated ticks), not just the one below.
+    const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);
+    try {
+        await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
+        // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an
+        // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).
+        const version = await getMeta(conn, 'schema_version');
+        const features = await getMeta(conn, 'features');
+        const wantFeatures = featureSignature(cfg, FEATURES);
+        let tokenizeOnlyRebuild = false;
+        if (version !== null && version !== dialect.schemaVersion || features !== null && features !== wantFeatures) {
+            // Indexing derives from presets, so a config edit rebuilding the cache must say so and
+            // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
+            if (version !== null && version !== dialect.schemaVersion) {
+                console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
+                // Close before clearCache deletes the files out from under the still-open handle.
+                closed = true;
+                await dialect.close(handle);
+                clearCache(cfg);
+                return connectWithDialect(cfg, dialect, keepBuilderOpen);
+            }
+            const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
+            // Only a dialect-owned carve-out (sqlite's tokenizer) skips the rebuild below with its own
+            // partial DDL swap; every other single- or multi-segment change takes it.
+            if (dialect.partialRebuild && await dialect.partialRebuild(handle, conn, cfg, changedKeys)) {
+                tokenizeOnlyRebuild = true;
+            } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
+                // First sight of a resolved weight identity: the model itself hasn't changed, so
+                // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.
+                console.error("sense: recorded the embedding model's resolved identity; vectors are unaffected");
+                await setMeta(conn, 'features', wantFeatures);
+            } else {
+                const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
+                console.error(`sense: config change (${changed}) rebuilds the index`);
+                closed = true;
+                await dialect.close(handle);
+                clearCache(cfg);
+                return connectWithDialect(cfg, dialect, keepBuilderOpen);
+            }
+        }
+        // A rebuild trigger independent of the signature comparison above (sqlite: the content
+        // table's own DDL no longer matches the resolved tokenizer).
+        const extraReason = dialect.extraRebuildReason ? dialect.extraRebuildReason(handle) : null;
+        if (extraReason !== null) {
+            console.error(`sense: ${extraReason}; rebuilding the index`);
+            closed = true;
+            await dialect.close(handle);
+            clearCache(cfg);
+            return connectWithDialect(cfg, dialect, keepBuilderOpen);
+        }
+        await dialect.ensureSchema(handle, conn, cfg);
+        let rebuildWarnings = [];
+        if (tokenizeOnlyRebuild && dialect.postSchemaRebuild) {
+            rebuildWarnings = await dialect.postSchemaRebuild(handle, conn, cfg, cfg.baseDir, wantFeatures);
+        }
+        // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
+        // Installed before build() -- that call is the one that races a watcher's transaction.
+        if (dialect.setDerivedBusyTimeout) {
+            var _ref;
+            const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
+            await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600000));
+        }
+        const { parsed, warnings } = await builder.build();
+        // A one-shot open is done reconciling for good; a watcher keeps `builder` alive across its run.
+        if (!keepBuilderOpen) await builder.close();
+        return {
+            handle,
+            conn,
+            cfg,
+            dbPath,
+            parsed,
+            warnings: [
+                ...rebuildWarnings,
+                ...warnings
+            ],
+            builder
+        };
+    } catch (err) {
+        await builder.close();
+        if (!closed) await dialect.close(handle);
+        throw err;
+    }
+}
+// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection
+// in the Store interface. The builder's pool closes right after the initial build.
+export async function openWithDialect(cfg, dialect) {
+    const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connectWithDialect(cfg, dialect, false);
+    const store = dialect.createStore(handle, conn, resolvedCfg);
+    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+    rekeyChunkText(conn, store);
+    return {
+        store,
+        cfg: resolvedCfg,
+        dbPath,
+        parsed,
+        warnings
+    };
+}
+// Same connect+build as openWithDialect, but keeps the builder's pool alive and hands it back
+// alongside the store, for a caller (a watcher) that reconciles repeatedly on this connection.
+export async function openWithBuilder(cfg, dialect) {
+    const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, builder } = await connectWithDialect(cfg, dialect, true);
+    const store = dialect.createStore(handle, conn, resolvedCfg);
+    rekeyChunkText(conn, store);
+    return {
+        store,
+        cfg: resolvedCfg,
+        dbPath,
+        parsed,
+        warnings,
+        builder
+    };
+}
+export async function docCount(store) {
+    const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');
+    return (await stmt.get()).n;
+}

package/dist/esm/store/open.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { Builder } from './builder.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Closed already unless `keepBuilderOpen` was set: a one-shot open reconciles once and is done\n  // with it, a watcher keeps calling build() on the same instance across its whole run.\n  builder: Builder;\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>, keepBuilderOpen: boolean): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await dialect.connect(dbPath, cfg);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): its pool must survive across every build() call this\n  // connection sees (a watcher's repeated ticks), not just the one below.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only a dialect-owned carve-out (sqlite's tokenizer) skips the rebuild below with its own\n      // partial DDL swap; every other single- or multi-segment change takes it.\n      if (dialect.partialRebuild && (await dialect.partialRebuild(handle, conn, cfg, changedKeys))) {\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n    }\n\n    // A rebuild trigger independent of the signature comparison above (sqlite: the content\n    // table's own DDL no longer matches the resolved tokenizer).\n    const extraReason = dialect.extraRebuildReason ? dialect.extraRebuildReason(handle) : null;\n    if (extraReason !== null) {\n      console.error(`sense: ${extraReason}; rebuilding the index`);\n      closed = true;\n      await dialect.close(handle);\n      clearCache(cfg);\n      return connectWithDialect(cfg, dialect, keepBuilderOpen);\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild && dialect.postSchemaRebuild) {\n      rebuildWarnings = await dialect.postSchemaRebuild(handle, conn, cfg, cfg.baseDir, wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    const { parsed, warnings } = await builder.build();\n    // A one-shot open is done reconciling for good; a watcher keeps `builder` alive across its run.\n    if (!keepBuilderOpen) await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings], builder };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connectWithDialect(cfg, dialect, false);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\n// Same connect+build as openWithDialect, but keeps the builder's pool alive and hands it back\n// alongside the store, for a caller (a watcher) that reconciles repeatedly on this connection.\nexport async function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & { builder: Builder }> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, builder } = await connectWithDialect(cfg, dialect, true);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, builder };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","FEATURES","createBuilder","clearCache","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","connectWithDialect","cfg","dialect","keepBuilderOpen","stateDir","baseDir","recursive","dbPath","filename","handle","conn","connect","closed","builder","reconcileDialect","exec","version","features","wantFeatures","tokenizeOnlyRebuild","schemaVersion","console","error","close","changedKeys","partialRebuild","size","has","changed","extraReason","extraRebuildReason","ensureSchema","rebuildWarnings","postSchemaRebuild","setDerivedBusyTimeout","recordedMaxMs","Number","Math","min","max","parsed","warnings","build","err","openWithDialect","resolvedCfg","store","createStore","openWithBuilder","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AACxC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA4B3F,eAAeC,mBAA2BC,GAAmB,EAAEC,OAA4B,EAAEC,eAAwB;IACnH,MAAMC,WAAWhB,KAAKa,IAAII,OAAO,EAAEf;IACnCH,UAAUiB,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAASnB,KAAKgB,UAAUF,QAAQM,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAMR,QAAQS,OAAO,CAACJ,QAAQN;IACvD,gGAAgG;IAChG,kGAAkG;IAClG,IAAIW,SAAS;IACb,6FAA6F;IAC7F,wEAAwE;IACxE,MAAMC,UAAUpB,cAAciB,MAAMT,KAAKA,IAAII,OAAO,EAAEH,QAAQY,gBAAgB;IAC9E,IAAI;QACF,MAAMJ,KAAKK,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMrB,QAAQe,MAAM;QACpC,MAAMO,WAAW,MAAMtB,QAAQe,MAAM;QACrC,MAAMQ,eAAe7B,iBAAiBY,KAAKT;QAC3C,IAAI2B,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAYd,QAAQkB,aAAa,IAAMH,aAAa,QAAQA,aAAaC,cAAe;YAC/G,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYd,QAAQkB,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFV,SAAS;gBACT,MAAMV,QAAQqB,KAAK,CAACd;gBACpBf,WAAWO;gBACX,OAAOD,mBAAmBC,KAAKC,SAASC;YAC1C;YACA,MAAMqB,cAAc3B,qBAAqBoB,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,2FAA2F;YAC3F,0EAA0E;YAC1E,IAAIhB,QAAQuB,cAAc,IAAK,MAAMvB,QAAQuB,cAAc,CAAChB,QAAQC,MAAMT,KAAKuB,cAAe;gBAC5FL,sBAAsB;YACxB,OAAO,IAAIK,YAAYE,IAAI,KAAK,KAAKF,YAAYG,GAAG,CAAC,YAAY7B,qBAAqBmB,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFG,QAAQC,KAAK,CAAC;gBACd,MAAM1B,QAAQc,MAAM,YAAYQ;YAClC,OAAO;gBACL,MAAMU,UAAU7B,cAAckB,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CG,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEM,QAAQ,oBAAoB,CAAC;gBACpEhB,SAAS;gBACT,MAAMV,QAAQqB,KAAK,CAACd;gBACpBf,WAAWO;gBACX,OAAOD,mBAAmBC,KAAKC,SAASC;YAC1C;QACF;QAEA,uFAAuF;QACvF,6DAA6D;QAC7D,MAAM0B,cAAc3B,QAAQ4B,kBAAkB,GAAG5B,QAAQ4B,kBAAkB,CAACrB,UAAU;QACtF,IAAIoB,gBAAgB,MAAM;YACxBR,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEO,YAAY,sBAAsB,CAAC;YAC3DjB,SAAS;YACT,MAAMV,QAAQqB,KAAK,CAACd;YACpBf,WAAWO;YACX,OAAOD,mBAAmBC,KAAKC,SAASC;QAC1C;QAEA,MAAMD,QAAQ6B,YAAY,CAACtB,QAAQC,MAAMT;QAEzC,IAAI+B,kBAA4B,EAAE;QAClC,IAAIb,uBAAuBjB,QAAQ+B,iBAAiB,EAAE;YACpDD,kBAAkB,MAAM9B,QAAQ+B,iBAAiB,CAACxB,QAAQC,MAAMT,KAAKA,IAAII,OAAO,EAAEa;QACpF;QAEA,wFAAwF;QACxF,uFAAuF;QACvF,IAAIhB,QAAQgC,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAMzC,QAAQe,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMR,QAAQgC,qBAAqB,CAACzB,QAAQC,MAAM2B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB;QACjG;QAEA,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAM5B,QAAQ6B,KAAK;QAChD,gGAAgG;QAChG,IAAI,CAACvC,iBAAiB,MAAMU,QAAQU,KAAK;QAEzC,OAAO;YAAEd;YAAQC;YAAMT;YAAKM;YAAQiC;YAAQC,UAAU;mBAAIT;mBAAoBS;aAAS;YAAE5B;QAAQ;IACnG,EAAE,OAAO8B,KAAK;QACZ,MAAM9B,QAAQU,KAAK;QACnB,IAAI,CAACX,QAAQ,MAAMV,QAAQqB,KAAK,CAACd;QACjC,MAAMkC;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeC,gBAAwB3C,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEO,MAAM,EAAEC,IAAI,EAAET,KAAK4C,WAAW,EAAEtC,MAAM,EAAEiC,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzC,mBAAmBC,KAAKC,SAAS;IAC5G,MAAM4C,QAAQ5C,QAAQ6C,WAAW,CAACtC,QAAQC,MAAMmC;IAChD,0FAA0F;IAC1FtD,eAAemB,MAAMoC;IACrB,OAAO;QAAEA;QAAO7C,KAAK4C;QAAatC;QAAQiC;QAAQC;IAAS;AAC7D;AAEA,8FAA8F;AAC9F,+FAA+F;AAC/F,OAAO,eAAeO,gBAAwB/C,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEO,MAAM,EAAEC,IAAI,EAAET,KAAK4C,WAAW,EAAEtC,MAAM,EAAEiC,MAAM,EAAEC,QAAQ,EAAE5B,OAAO,EAAE,GAAG,MAAMb,mBAAmBC,KAAKC,SAAS;IACrH,MAAM4C,QAAQ5C,QAAQ6C,WAAW,CAACtC,QAAQC,MAAMmC;IAChDtD,eAAemB,MAAMoC;IACrB,OAAO;QAAEA;QAAO7C,KAAK4C;QAAatC;QAAQiC;QAAQC;QAAU5B;IAAQ;AACtE;AAEA,OAAO,eAAeoC,SAASH,KAAY;IACzC,MAAMI,OAAO,MAAMJ,MAAMK,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}

package/dist/esm/store/reconcile.d.ts

@@ -1,6 +1,7 @@
 import type { Config } from '../config/index.js';
+import type { ParsePool } from '../scan/pool.js';
 import type { Connection, ReconcileDialect } from './types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool): Promise<{
     parsed: number;
     warnings: string[];
 }>;

package/dist/esm/store/reconcile.js

@@ -18,7 +18,7 @@ const CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-export async function reconcile(conn, cfg, baseDir, dialect) {
+export async function reconcile(conn, cfg, baseDir, dialect, pool) {
     const files = listFiles(cfg, baseDir);
     const currentSet = new Set(files.map((f)=>f.relPath));
     const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
@@ -41,7 +41,9 @@ export async function reconcile(conn, cfg, baseDir, dialect) {
     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
     // reconciles stay silent (progress() has a threshold).
     const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick, {
+        pool
+    });
     report.finish();
     for (const col of newColumns)seenColumns.add(col);
     const allColumns = [

package/dist/esm/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      for (const col of newColumns) if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}${dialect.columnTypeSuffix()}`);\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg);\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      if (touched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          touched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      if (touched.length > 0) for (const feature of features) await feature.remove?.(conn, touched, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  if (dialect.recordDuration) await dialect.recordDuration(conn, durationMs);\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","files","currentSet","map","f","relPath","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","feature","present","exec","columnTypeSuffix","rows","doc","ctimeMs","parseError","data","runBatch","reconcileContent","presetRows","presetName","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAC/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACrD,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB;IACvG,MAAMC,QAAQb,UAAUU,KAAKC;IAC7B,MAAMG,aAAa,IAAIP,IAAIM,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaL,GAAG,CAAC,CAACS,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACV,WAAWc,GAAG,CAACJ,EAAEC,IAAI,GAAGV,GAAG,CAAC,CAACS,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYhB,MAAMc,MAAM,CAAC,CAACX;QAC9B,MAAMc,MAAMR,SAASS,GAAG,CAACf,EAAEC,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWzC,eAAeY;IAChC,MAAM8B,cAAc,MAAMrC,WAAWM;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAMgC,SAAS1C,SAAS,mBAAmB8B,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAM1C,aAAa2B,WAAWU,UAAU7B,KAAK8B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG5B,QAAQsC,gBAAgB,CAACD,WAAWb,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMe,kBAAkBF,WAAWtB,MAAM,CAAC,CAACyB,IAAM9C,yBAAyBsB,GAAG,CAACwB,MAAM,CAACnD,iBAAiB2B,GAAG,CAACwB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBpC,GAAG,CAACX,YAAYkD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBpC,GAAG,CAAC,IAAM,KAAKuC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLxB,MAAM,CAAC,CAACyB,IAAMA,MAAM,QACpBrC,GAAG,CAAC,CAACqC,IAAM,GAAGhD,WAAWgD,GAAG,YAAY,EAAEhD,WAAWgD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ1B,UAAUF,MAAM,CAAC,CAACX,IAAM,CAACM,SAASM,GAAG,CAACZ,EAAEC,OAAO,GAAGF,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACpF,MAAMuC,QAAwB;QAAE3C;QAAO4C,UAAUd,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO;QAAGsC;QAAO7B;IAAS;IACnG,MAAMiC,WAAW,IAAIpD,IAAIgD;IACzB,MAAMK,mBAAmBjB,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO,EAAEU,MAAM,CAAC,CAACkC,IAAM,CAACF,SAAS/B,GAAG,CAACiC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIpC;WAAakC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM5D,gBACJI,MACA;YA4CgEyD,iBAKxBA;QAhDtC,iGAAiG;QACjG,sFAAsF;QACtF,MAAMC,UAAU,MAAMhE,WAAWM;QACjC,KAAK,MAAMsC,OAAOH,WAAY,IAAI,CAACuB,QAAQvC,GAAG,CAACmB,MAAM,MAAMtC,KAAK2D,IAAI,CAAC,CAAC,mCAAmC,EAAEhE,WAAW2C,OAAOnC,QAAQyD,gBAAgB,IAAI;QAEzJ,IAAI1B,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMkC,OAAO3B,WAAW5B,GAAG,CAAC,CAACwD,MAC3BpB,gBAAgBpC,GAAG,CAAC,CAACgC;wBAOZwB;oBANP,IAAIxB,QAAQ,QAAQ,OAAOwB,IAAItD,OAAO;oBACtC,IAAI8B,QAAQ,UAAU,OAAOwB,IAAItC,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAOwB,IAAIC,OAAO;oBACxC,IAAIzB,QAAQ,SAAS,OAAOwB,IAAIpC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAOwB,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC3B,IAAI,cAAbwB,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAM9D,KAAKkE,QAAQ,CAACtB,WAAWiB;QACjC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAM1D,QAAQgE,gBAAgB,CAACnE,MAAMqD,SAASnB,YAAYa,OAAO9C;QAEjE,0FAA0F;QAC1F,kEAAkE;QAClE,IAAIoD,QAAQ1B,MAAM,GAAG,GACnB,MAAM3B,KAAKkE,QAAQ,CACjB,6CACAb,QAAQ/C,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAE1B,MAAMgB,aAA0B,EAAE;QAClC,KAAK,MAAMN,OAAO5B,WAAY,KAAK,MAAMmC,cAAcP,IAAIQ,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACT,IAAItD,OAAO;YAAE6D;SAAW;QAC7G,IAAID,WAAWzC,MAAM,GAAG,GAAG,MAAM3B,KAAKkE,QAAQ,CAAC,2DAA2DE;QAE1G,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAInD,SAASU,MAAM,GAAG,GACpB,MAAM3B,KAAKkE,QAAQ,CACjB,4CACAjD,SAASX,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAG3B,IAAIC,QAAQ1B,MAAM,GAAG,GAAG,KAAK,MAAM8B,WAAW3B,SAAU,QAAM2B,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBzD,MAAMqD,SAASN;QAC9F,KAAK,MAAMU,WAAW3B,SAAU;gBAExB2B;YADN,MAAMgB,iBAAiCvC,WAAW5B,GAAG,CAAC,CAACwD,MAAS,CAAA;oBAAE9C,MAAM8C,IAAItD,OAAO;oBAAEkE,WAAWZ,IAAIY,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBzD,MAAMyE,gBAAgB1B;QAC9C;QACA,KAAK,MAAMU,WAAW3B,SAAU,QAAM2B,0BAAAA,QAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,SAAyBzD,MAAM+C;IACvE,GACA5C,QAAQ2E,SAAS;IAGnB,MAAMC,aAAaxB,KAAKC,GAAG,KAAKF;IAChC,IAAInD,QAAQ6E,cAAc,EAAE,MAAM7E,QAAQ6E,cAAc,CAAChF,MAAM+E;IAE/D,OAAO;QAAEnD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool });\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      for (const col of newColumns) if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}${dialect.columnTypeSuffix()}`);\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg);\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      if (touched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          touched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      if (touched.length > 0) for (const feature of features) await feature.remove?.(conn, touched, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  if (dialect.recordDuration) await dialect.recordDuration(conn, durationMs);\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","pool","files","currentSet","map","f","relPath","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","feature","present","exec","columnTypeSuffix","rows","doc","ctimeMs","parseError","data","runBatch","reconcileContent","presetRows","presetName","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACrD,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB;IACzH,MAAMC,QAAQd,UAAUU,KAAKC;IAC7B,MAAMI,aAAa,IAAIR,IAAIO,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe,MAAMV,KAAKW,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaL,GAAG,CAAC,CAACS,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACV,WAAWc,GAAG,CAACJ,EAAEC,IAAI,GAAGV,GAAG,CAAC,CAACS,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYhB,MAAMc,MAAM,CAAC,CAACX;QAC9B,MAAMc,MAAMR,SAASS,GAAG,CAACf,EAAEC,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAW1C,eAAeY;IAChC,MAAM+B,cAAc,MAAMtC,WAAWM;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAMiC,SAAS3C,SAAS,mBAAmB+B,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAM3C,aAAa4B,WAAWU,UAAU9B,KAAK+B,aAAaC,OAAOI,IAAI,EAAE;QAAEjC;IAAK;IACjI6B,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG7B,QAAQuC,gBAAgB,CAACD,WAAWb,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMe,kBAAkBF,WAAWtB,MAAM,CAAC,CAACyB,IAAM/C,yBAAyBuB,GAAG,CAACwB,MAAM,CAACpD,iBAAiB4B,GAAG,CAACwB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBpC,GAAG,CAACZ,YAAYmD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBpC,GAAG,CAAC,IAAM,KAAKuC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLxB,MAAM,CAAC,CAACyB,IAAMA,MAAM,QACpBrC,GAAG,CAAC,CAACqC,IAAM,GAAGjD,WAAWiD,GAAG,YAAY,EAAEjD,WAAWiD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ1B,UAAUF,MAAM,CAAC,CAACX,IAAM,CAACM,SAASM,GAAG,CAACZ,EAAEC,OAAO,GAAGF,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACpF,MAAMuC,QAAwB;QAAE3C;QAAO4C,UAAUd,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO;QAAGsC;QAAO7B;IAAS;IACnG,MAAMiC,WAAW,IAAIrD,IAAIiD;IACzB,MAAMK,mBAAmBjB,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO,EAAEU,MAAM,CAAC,CAACkC,IAAM,CAACF,SAAS/B,GAAG,CAACiC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIpC;WAAakC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM7D,gBACJI,MACA;YA4CgE0D,iBAKxBA;QAhDtC,iGAAiG;QACjG,sFAAsF;QACtF,MAAMC,UAAU,MAAMjE,WAAWM;QACjC,KAAK,MAAMuC,OAAOH,WAAY,IAAI,CAACuB,QAAQvC,GAAG,CAACmB,MAAM,MAAMvC,KAAK4D,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAW4C,OAAOpC,QAAQ0D,gBAAgB,IAAI;QAEzJ,IAAI1B,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMkC,OAAO3B,WAAW5B,GAAG,CAAC,CAACwD,MAC3BpB,gBAAgBpC,GAAG,CAAC,CAACgC;wBAOZwB;oBANP,IAAIxB,QAAQ,QAAQ,OAAOwB,IAAItD,OAAO;oBACtC,IAAI8B,QAAQ,UAAU,OAAOwB,IAAItC,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAOwB,IAAIC,OAAO;oBACxC,IAAIzB,QAAQ,SAAS,OAAOwB,IAAIpC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAOwB,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC3B,IAAI,cAAbwB,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAM/D,KAAKmE,QAAQ,CAACtB,WAAWiB;QACjC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAM3D,QAAQiE,gBAAgB,CAACpE,MAAMsD,SAASnB,YAAYa,OAAO/C;QAEjE,0FAA0F;QAC1F,kEAAkE;QAClE,IAAIqD,QAAQ1B,MAAM,GAAG,GACnB,MAAM5B,KAAKmE,QAAQ,CACjB,6CACAb,QAAQ/C,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAE1B,MAAMgB,aAA0B,EAAE;QAClC,KAAK,MAAMN,OAAO5B,WAAY,KAAK,MAAMmC,cAAcP,IAAIQ,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACT,IAAItD,OAAO;YAAE6D;SAAW;QAC7G,IAAID,WAAWzC,MAAM,GAAG,GAAG,MAAM5B,KAAKmE,QAAQ,CAAC,2DAA2DE;QAE1G,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAInD,SAASU,MAAM,GAAG,GACpB,MAAM5B,KAAKmE,QAAQ,CACjB,4CACAjD,SAASX,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAG3B,IAAIC,QAAQ1B,MAAM,GAAG,GAAG,KAAK,MAAM8B,WAAW3B,SAAU,QAAM2B,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiB1D,MAAMsD,SAASN;QAC9F,KAAK,MAAMU,WAAW3B,SAAU;gBAExB2B;YADN,MAAMgB,iBAAiCvC,WAAW5B,GAAG,CAAC,CAACwD,MAAS,CAAA;oBAAE9C,MAAM8C,IAAItD,OAAO;oBAAEkE,WAAWZ,IAAIY,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgB1D,MAAM0E,gBAAgB1B;QAC9C;QACA,KAAK,MAAMU,WAAW3B,SAAU,QAAM2B,0BAAAA,QAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,SAAyB1D,MAAMgD;IACvE,GACA7C,QAAQ4E,SAAS;IAGnB,MAAMC,aAAaxB,KAAKC,GAAG,KAAKF;IAChC,IAAIpD,QAAQ8E,cAAc,EAAE,MAAM9E,QAAQ8E,cAAc,CAACjF,MAAMgF;IAE/D,OAAO;QAAEnD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/{sqlite/signature.d.ts → signature.d.ts}

@@ -1,6 +1,3 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
 export declare function changedSignatureKeys(before: string, after: string): Set<string>;
 export declare function embedIdentityAdopted(before: string, after: string): boolean;
 export declare function signatureDiff(before: string, after: string): string;
-export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;

package/dist/esm/store/{sqlite/signature.js → signature.js}

@@ -1,10 +1,6 @@
-import { contentTokenize } from '../../config/index.js';
-import { listFiles, parseFile } from '../../scan/index.js';
-import { BEGIN_WRITE, withTransaction } from '../transaction.js';
-import { contentRow, INSERT_CONTENT_SQL } from './reconcile.js';
-// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:
-// global features, embed provider, tokenize, then one segment per preset). Never called from
-// reconcile() itself.
+// Feature-signature comparison for open()'s rebuild-vs-adopt decision (config.featureSignature's
+// format: global features, embed provider, tokenize, then one segment per preset). Pure string
+// comparison, no SQL, shared by every store's open dialect.
 // Segment keys that moved between two feature signatures.
 export function changedSignatureKeys(before, after) {
     const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
@@ -37,22 +33,3 @@ export function signatureDiff(before, after) {
         ...changed
     ].map(label).join(', ');
 }
-// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
-// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.
-export async function rebuildContentTable(conn, cfg, baseDir) {
-    const stmt = await conn.prepare('SELECT "path" FROM frontmatter');
-    const known = new Set((await stmt.all()).map((r)=>r.path));
-    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
-    const segmenting = contentTokenize(cfg) === undefined;
-    const warnings = [];
-    const rows = [];
-    for (const file of files){
-        const { doc, warnings: fileWarnings } = parseFile(file);
-        warnings.push(...fileWarnings);
-        rows.push(contentRow(doc, segmenting));
-    }
-    await withTransaction(conn, async ()=>{
-        if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
-    }, BEGIN_WRITE);
-    return warnings;
-}

package/dist/esm/store/signature.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/signature.ts"],"sourcesContent":["// Feature-signature comparison for open()'s rebuild-vs-adopt decision (config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset). Pure string\n// comparison, no SQL, shared by every store's open dialect.\n\n// Segment keys that moved between two feature signatures.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n"],"names":["changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedIdentityAdopted","embedPart","find","p","undefined","at","indexOf","signatureDiff","label","size"],"mappings":"AAAA,iGAAiG;AACjG,+FAA+F;AAC/F,4DAA4D;AAE5D,0DAA0D;AAC1D,OAAO,SAASA,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMG,QAAQ,CAACC,MAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,CAACR,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMS,IAAIJ,MAAMR;IAChB,MAAMa,IAAIL,MAAMP;IAChB,MAAMa,UAAU,IAAIC;IACpB,KAAK,MAAM,CAACC,KAAKC,IAAI,IAAIJ,EAAG,IAAID,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;IAChE,KAAK,MAAMA,OAAOJ,EAAEQ,IAAI,GAAI,IAAI,CAACP,EAAEQ,GAAG,CAACL,MAAMF,QAAQK,GAAG,CAACH;IACzD,OAAOF;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASQ,qBAAqBtB,MAAc,EAAEC,KAAa;IAChE,MAAMsB,YAAY,CAACd,MAAgBA,IAAIJ,KAAK,CAAC,KAAKmB,IAAI,CAAC,CAACC,IAAMA,EAAErB,UAAU,CAAC;IAC3E,MAAMS,IAAIU,UAAUvB;IACpB,MAAMY,IAAIW,UAAUtB;IACpB,IAAIY,MAAMa,aAAad,MAAMc,WAAW,OAAO;IAC/C,MAAMC,KAAKf,EAAEgB,OAAO,CAAC;IACrB,OAAOf,EAAEe,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKf,EAAEN,KAAK,CAAC,GAAGqB,QAAQd;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASgB,cAAc7B,MAAc,EAAEC,KAAa;IACzD,MAAMa,UAAUf,qBAAqBC,QAAQC;IAC7C,MAAM6B,QAAQ,CAACd,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIZ,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOQ,QAAQiB,IAAI,KAAK,IAAI,aAAa;WAAIjB;KAAQ,CAACH,GAAG,CAACmB,OAAOvB,IAAI,CAAC;AACxE"}

package/dist/esm/store/sqlite/open.d.ts

@@ -1,13 +1,13 @@
+import { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "18";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
+export declare const SCHEMA_VERSION = "19";
+export type { OpenResult };
+interface SqliteHandle {
+    db: DatabaseSync;
+    tokenize: string;
 }
+export declare const sqliteOpenDialect: OpenDialect<SqliteHandle>;
 export declare function openSqlite(cfg: ResolvedConfig): Promise<OpenResult>;
-export declare function docCount(store: Store): Promise<number>;