sensemaking 0.19.2 → 0.21.0

package/dist/esm/store/duckdb/open.js

@@ -1,12 +1,8 @@
-import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
-import { featureSignature, STATE_DIR } from '../../config/index.js';
-import { rekeyChunkText } from '../../embed/handoff.js';
+import { featureSignature } from '../../config/index.js';
 import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
-import { clearCache } from '../cache.js';
-import { reconcile } from '../reconcile.js';
+import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
 import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
@@ -16,10 +12,10 @@ import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.duckdb';
 // Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
 // The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
-export const SCHEMA_VERSION = '2';
+export const SCHEMA_VERSION = '3';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
-// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.
-async function ensureSchema(conn, cfg) {
+// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
+async function ensureSchema(_handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
@@ -36,12 +32,12 @@ async function ensureSchema(conn, cfg) {
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
-async function connect(cfg) {
-    const stateDir = join(cfg.baseDir, STATE_DIR);
-    mkdirSync(stateDir, {
-        recursive: true
-    });
-    const dbPath = join(stateDir, DB_FILENAME);
+// Order matters: the connection must be gone before the instance closes the WAL.
+async function close(handle) {
+    handle.duckdb.disconnectSync();
+    handle.instance.closeSync();
+}
+async function connect(dbPath, _cfg) {
     // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports
     // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.
     let DuckDBInstance;
@@ -65,57 +61,39 @@ async function connect(cfg) {
         instance === null || instance === void 0 ? void 0 : instance.closeSync();
         throw new SenseError('STORE_DEPENDENCY_MISSING', `store "duckdb" failed to open ${dbPath}: ${err.message}`);
     }
-    // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL
-    // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.
-    let closed = false;
+    // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.
     try {
         await registerFunctions(duckdb);
         const conn = createConnection(duckdb);
-        await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
-        const version = await getMeta(conn, 'schema_version');
-        const features = await getMeta(conn, 'features');
-        const wantFeatures = featureSignature(cfg, FEATURES);
-        if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
-            // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
-            // rebuild instead, same as sqlite's open() (see its comment for why this is announced).
-            const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
-            console.error(`sense: ${reason}; rebuilding the index`);
-            // Close before clearCache deletes the files underneath the still-open instance.
-            closed = true;
-            duckdb.disconnectSync();
-            instance.closeSync();
-            clearCache(cfg);
-            return connect(cfg);
-        }
-        await ensureSchema(conn, cfg);
-        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, duckdbDialect);
         return {
-            instance,
-            duckdb,
-            conn,
-            cfg,
-            dbPath,
-            parsed,
-            warnings
+            handle: {
+                instance,
+                duckdb
+            },
+            conn
         };
     } catch (err) {
-        if (!closed) {
-            duckdb.disconnectSync();
-            instance === null || instance === void 0 ? void 0 : instance.closeSync();
-        }
+        await close({
+            instance,
+            duckdb
+        });
         throw err;
     }
 }
+// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.
+export const duckdbOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: duckdbDialect,
+    connect,
+    close,
+    // duckdb's file lock spans the connection's life, and each platform words the refusal its own
+    // way: posix "Could not set lock on file ... Conflicting lock is held in <exe> (PID n)", Windows
+    // "Cannot open file ... being used by another process". Both are the same condition.
+    isLocked: (err)=>/Could not set lock on file|being used by another process/.test(err.message),
+    ensureSchema,
+    createStore: (handle, conn)=>createStore(handle.instance, handle.duckdb, conn)
+};
 export async function openDuckdb(cfg) {
-    const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
-    const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);
-    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-    rekeyChunkText(conn, store);
-    return {
-        store: store,
-        cfg: resolvedCfg,
-        dbPath,
-        parsed,
-        warnings
-    };
+    return openWithDialect(cfg, duckdbOpenDialect);
 }

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, duckdbDialect);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","reconcile","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,8LAA8L;AAC9L,2JAA2J;AAC3J,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMX,QAAQW,MAAM,sBAAuB,MAAM,MAAMV,QAAQU,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMT,QAAQW,MAAM,gBAAiB,MAAM,MAAMV,QAAQU,MAAM,YAAYpB,iBAAiBqB,KAAKf;AACxG;AAEA,eAAeoB,QAAQL,GAAmB;IACxC,MAAMM,WAAW5B,KAAKsB,IAAIO,OAAO,EAAE3B;IACnCH,UAAU6B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS/B,KAAK4B,UAAUV;IAE9B,gIAAgI;IAChI,qLAAqL;IACrL,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe5B,YAAY,MAAM4B;QACrC,MAAM,IAAI5B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEQ,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIlC,WAAW,4BAA4B,CAAC,8BAA8B,EAAE0B,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAM/B,QAAQW,MAAM;QACpC,MAAMqB,WAAW,MAAMhC,QAAQW,MAAM;QACrC,MAAMsB,eAAe1C,iBAAiBqB,KAAKf;QAC3C,IAAI,AAACkC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB/B,WAAWc;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMxC,UAAUY,MAAMC,KAAKA,IAAIO,OAAO,EAAEd;QACrE,OAAO;YAAEqB;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,MAAM8B,QAAQnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;IAClF,0FAA0F;IAC1F1B,eAAekB,MAAM+B;IACrB,OAAO;QAAEA,OAAOA;QAAO9B,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}

package/dist/esm/store/duckdb/reconcile.js

@@ -1,4 +1,6 @@
 import { SenseError } from '../../errors.js';
+import { quoteIdent } from '../shared.js';
+import { markContentStale } from './lexical.js';
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
 // store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here
 // unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter
@@ -22,6 +24,18 @@ async function reconcileContent(conn, touched, docs, _delta) {
             p
         ]));
     if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));
+    // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
+    // not here (lexical.ts's FtsIndexState).
+    markContentStale(conn);
+}
+// DuckDB rejects more than one ALTER command per statement ("Parser Error: Only one ALTER
+// command per statement is supported", measured), so every name's clause joins into one string
+// and runs as a single exec() -- one column-add "leg" through the driver instead of `names.length`.
+// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()
+// produces for one key across files.
+async function addColumns(conn, names) {
+    if (names.length === 0) return;
+    await conn.exec(names.map((name)=>`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));
 }
 export const duckdbDialect = {
     beginMode: ()=>'BEGIN',
@@ -30,8 +44,6 @@ export const duckdbDialect = {
             throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
         }
     },
-    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold
-    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.
-    columnTypeSuffix: ()=>' VARIANT',
+    addColumns,
     reconcileContent
 };

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n  // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n  columnTypeSuffix: () => ' VARIANT',\n  reconcileContent,\n};\n"],"names":["SenseError","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","duckdbDialect","beginMode","checkColumnLimit","count","columnTypeSuffix"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAK7C,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIL,KAAKE,MAAM,GAAG,GAAG,MAAMJ,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;AACxE;AAEA,OAAO,MAAMiB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQb,yBAAyB;YACnC,MAAM,IAAIT,WAAW,gBAAgB,CAAC,uBAAuB,EAAEsB,MAAM,8CAA8C,EAAEb,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACA,yFAAyF;IACzF,2FAA2F;IAC3Fc,kBAAkB,IAAM;IACxBb;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","quoteIdent","markContentStale","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","addColumns","names","exec","name","join","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIL,KAAKE,MAAM,GAAG,GAAG,MAAMJ,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;IACtE,6FAA6F;IAC7F,yCAAyC;IACzCF,iBAAiBW;AACnB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeQ,WAAWR,IAAgB,EAAES,KAAe;IACzD,IAAIA,MAAML,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKU,IAAI,CAACD,MAAMH,GAAG,CAAC,CAACK,OAAS,CAAC,mCAAmC,EAAEvB,WAAWuB,MAAM,QAAQ,CAAC,EAAEC,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMC,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQlB,yBAAyB;YACnC,MAAM,IAAIX,WAAW,gBAAgB,CAAC,uBAAuB,EAAE6B,MAAM,8CAA8C,EAAElB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAU;IACAT;AACF,EAAE"}

package/dist/esm/store/duckdb/store.d.ts

@@ -1,5 +1,4 @@
 import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store;

package/dist/esm/store/duckdb/store.js

@@ -1,12 +1,10 @@
 import { STORE_DIMS } from '../../embed/types.js';
-import { reconcile } from '../reconcile.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
 import { storeRowToJs } from './connection.js';
 import { fieldStats } from './fieldStats.js';
 import { createLexicalIndex } from './lexical.js';
-import { duckdbDialect } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 // Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native
 // FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.
@@ -16,9 +14,9 @@ export const CAPABILITIES = new Set([
     'vectors',
     'sql-functions'
 ]);
-// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.
+// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.
 // The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.
-export function createStore(instance, duckdb, conn, cfg, baseDir) {
+export function createStore(instance, duckdb, conn) {
     const lex = createLexicalIndex(conn);
     return {
         name: 'duckdb',
@@ -35,13 +33,6 @@ export function createStore(instance, duckdb, conn, cfg, baseDir) {
         async transaction (fn) {
             return withTransaction(conn, fn);
         },
-        async reconcile () {
-            const result = await reconcile(conn, cfg, baseDir, duckdbDialect);
-            // content may have changed; the fts index is rebuilt lazily, on the next lexical query
-            // that needs it, not here (see lexical.ts's FtsIndexState).
-            lex.markStale();
-            return result;
-        },
         docs: {
             async columns () {
                 return [

package/dist/esm/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir, duckdbDialect);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","duckdbDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB,EAAE;AAEjH,8JAA8J;AAC9J,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAM7B;YACJ,MAAM8B,SAAS,MAAM9B,UAAUiB,MAAMC,KAAKC,SAASX;YACnD,uFAAuF;YACvF,4DAA4D;YAC5DY,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMlB,YAAYyC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI3C,YAAY6C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMlB,YAAYgD,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB,EAAE;AAEjH,mKAAmK;AACnK,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,MAAMC,MAAMV,mBAAmBS;IAC/B,OAAO;QACLE,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBc,MAAMU;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWe;iBAAO;YACtC;YACAV,YAAY,CAACsB,SAASC,aAAevB,WAAWU,MAAMY,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOd,IAAIc,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAM7B,YAAYY;YAC3BkB,cAAc,CAACC,OAASzB,iBAAiBK,QAAQC,MAAMhB,YAAYmC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,QAAQsB,IAAIrC,YAAYuC,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAASlC,YAAYM,QAAQC,MAAMhB,YAAY0C,MAAMC;YACrEC,WAAW,CAACF,OAASvC,aAAaa,MAAM0B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAMxB,SAAQD,GAAW;gBACvB,MAAM0B,WAAW,MAAMhC,OAAOO,OAAO,CAACD;gBACtC,OAAO;oBACLO,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEjC,MAAM6B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAMtD,aAAa8B,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFjD,OAAOkD,cAAc;YACrBnD,SAASoD,SAAS;QACpB;IACF;AACF"}

package/dist/esm/store/embed-scope.d.ts

@@ -0,0 +1,2 @@
+export type EmbedChangeKind = 'model' | 'chunk';
+export declare function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null;

package/dist/esm/store/embed-scope.js

@@ -0,0 +1,28 @@
+// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,
+// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.
+// One embed segment's provider/model/chunkVersion, ignoring any trailing `@identity`. `embed:off`
+// and any segment not in that exact shape return null, so the caller falls back to a full rebuild.
+function parseEmbedSegment(sig) {
+    const part = sig.split('|').find((p)=>p.startsWith('embed:'));
+    if (part === undefined || part === 'embed:off') return null;
+    const at = part.indexOf('@');
+    const body = at === -1 ? part : part.slice(0, at);
+    const match = /^embed:(static|openai|cohere):(.+):(chunk:v\d+(?::\d+)?)$/.exec(body);
+    if (!match) return null;
+    const [, provider, model, chunkVersion] = match;
+    return {
+        provider,
+        model,
+        chunkVersion
+    };
+}
+// 'chunk' when chunkTokens or the chunk version moved (boundaries changed, embeddings rebuild);
+// 'model' when only the provider or model moved (rows unchanged, values stale); null otherwise.
+export function classifyEmbedChange(before, after) {
+    const b = parseEmbedSegment(before);
+    const a = parseEmbedSegment(after);
+    if (b === null || a === null) return null;
+    if (b.chunkVersion !== a.chunkVersion) return 'chunk';
+    if (b.provider !== a.provider || b.model !== a.model) return 'model';
+    return null;
+}

package/dist/esm/store/embed-scope.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/embed-scope.ts"],"sourcesContent":["// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,\n// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.\n\nexport type EmbedChangeKind = 'model' | 'chunk';\n\ninterface EmbedSegment {\n  provider: string;\n  model: string;\n  chunkVersion: string;\n}\n\n// One embed segment's provider/model/chunkVersion, ignoring any trailing `@identity`. `embed:off`\n// and any segment not in that exact shape return null, so the caller falls back to a full rebuild.\nfunction parseEmbedSegment(sig: string): EmbedSegment | null {\n  const part = sig.split('|').find((p) => p.startsWith('embed:'));\n  if (part === undefined || part === 'embed:off') return null;\n  const at = part.indexOf('@');\n  const body = at === -1 ? part : part.slice(0, at);\n  const match = /^embed:(static|openai|cohere):(.+):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n  if (!match) return null;\n  const [, provider, model, chunkVersion] = match;\n  return { provider, model, chunkVersion };\n}\n\n// 'chunk' when chunkTokens or the chunk version moved (boundaries changed, embeddings rebuild);\n// 'model' when only the provider or model moved (rows unchanged, values stale); null otherwise.\nexport function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null {\n  const b = parseEmbedSegment(before);\n  const a = parseEmbedSegment(after);\n  if (b === null || a === null) return null;\n  if (b.chunkVersion !== a.chunkVersion) return 'chunk';\n  if (b.provider !== a.provider || b.model !== a.model) return 'model';\n  return null;\n}\n"],"names":["parseEmbedSegment","sig","part","split","find","p","startsWith","undefined","at","indexOf","body","slice","match","exec","provider","model","chunkVersion","classifyEmbedChange","before","after","b","a"],"mappings":"AAAA,+FAA+F;AAC/F,oFAAoF;AAUpF,kGAAkG;AAClG,mGAAmG;AACnG,SAASA,kBAAkBC,GAAW;IACpC,MAAMC,OAAOD,IAAIE,KAAK,CAAC,KAAKC,IAAI,CAAC,CAACC,IAAMA,EAAEC,UAAU,CAAC;IACrD,IAAIJ,SAASK,aAAaL,SAAS,aAAa,OAAO;IACvD,MAAMM,KAAKN,KAAKO,OAAO,CAAC;IACxB,MAAMC,OAAOF,OAAO,CAAC,IAAIN,OAAOA,KAAKS,KAAK,CAAC,GAAGH;IAC9C,MAAMI,QAAQ,4DAA4DC,IAAI,CAACH;IAC/E,IAAI,CAACE,OAAO,OAAO;IACnB,MAAM,GAAGE,UAAUC,OAAOC,aAAa,GAAGJ;IAC1C,OAAO;QAAEE;QAAUC;QAAOC;IAAa;AACzC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,OAAO,SAASC,oBAAoBC,MAAc,EAAEC,KAAa;IAC/D,MAAMC,IAAIpB,kBAAkBkB;IAC5B,MAAMG,IAAIrB,kBAAkBmB;IAC5B,IAAIC,MAAM,QAAQC,MAAM,MAAM,OAAO;IACrC,IAAID,EAAEJ,YAAY,KAAKK,EAAEL,YAAY,EAAE,OAAO;IAC9C,IAAII,EAAEN,QAAQ,KAAKO,EAAEP,QAAQ,IAAIM,EAAEL,KAAK,KAAKM,EAAEN,KAAK,EAAE,OAAO;IAC7D,OAAO;AACT"}

package/dist/esm/store/feature-scope.d.ts

@@ -0,0 +1,8 @@
+import type { FeatureName } from '../config/index.js';
+export declare const NARROW_FEATURE_TABLE: Readonly<Record<string, string>>;
+export interface FeatureToggle {
+    name: FeatureName;
+    turnedOn: boolean;
+}
+export declare function isFeatureOnlyChange(changedKeys: ReadonlySet<string>): boolean;
+export declare function classifyFeatureToggles(before: string, after: string, changedKeys: ReadonlySet<string>): FeatureToggle[] | null;

package/dist/esm/store/feature-scope.js

@@ -0,0 +1,49 @@
+// Decomposition of the `feature:<name>:<on|off>` featureSignature segments (config/access.ts's
+// featureSignature), for open()'s narrow feature-toggle invalidation routing.
+// Every narrow-supported feature but rank owns one dedicated table, dropped and (turning on)
+// fully re-derived. rank has no table -- it writes PageRank into frontmatter's own `_rank`
+// column -- so builder.ts handles it, and the links/rank pair, as special cases outside this map.
+export const NARROW_FEATURE_TABLE = {
+    tags: 'tags',
+    sections: 'sections',
+    links: 'links'
+};
+// links feeds rank's afterReconcile pass, so a links toggle always co-changes the `feature:rank`
+// segment too (featureEnabled's dependency); both are narrow-supported and builder.ts applies
+// them together. A changed set naming any other feature falls through to the existing full clear.
+const NARROW_FEATURE_NAMES = new Set([
+    ...Object.keys(NARROW_FEATURE_TABLE),
+    'rank'
+]);
+// Every changed key is a `feature:<name>` segment for a narrow-supported feature.
+export function isFeatureOnlyChange(changedKeys) {
+    return changedKeys.size > 0 && [
+        ...changedKeys
+    ].every((key)=>key.startsWith('feature:') && NARROW_FEATURE_NAMES.has(key.slice('feature:'.length)));
+}
+// One feature segment's old on/off, read back out of a stored `feature:<name>:on|off` segment.
+// Anything not in that exact shape returns null so the caller falls back to a full rebuild.
+function parseOnOff(sig, name) {
+    const part = sig.split('|').find((p)=>p.startsWith(`feature:${name}:`));
+    if (part === undefined) return null;
+    const fields = part.split(':');
+    if (fields.length !== 3 || fields[2] !== 'on' && fields[2] !== 'off') return null;
+    return fields[2] === 'on';
+}
+// Old-vs-new on/off for every changed feature key, or null when a segment can't be parsed back
+// out of `before` or `after`, or when it parses to the same state on both sides (should never
+// happen for a genuinely changed key) -- either way the caller takes the full rebuild.
+export function classifyFeatureToggles(before, after, changedKeys) {
+    const toggles = [];
+    for (const key of changedKeys){
+        const name = key.slice('feature:'.length);
+        const wasOn = parseOnOff(before, name);
+        const isOn = parseOnOff(after, name);
+        if (wasOn === null || isOn === null || wasOn === isOn) return null;
+        toggles.push({
+            name,
+            turnedOn: isOn
+        });
+    }
+    return toggles;
+}

package/dist/esm/store/feature-scope.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/feature-scope.ts"],"sourcesContent":["import type { FeatureName } from '../config/index.ts';\n\n// Decomposition of the `feature:<name>:<on|off>` featureSignature segments (config/access.ts's\n// featureSignature), for open()'s narrow feature-toggle invalidation routing.\n\n// Every narrow-supported feature but rank owns one dedicated table, dropped and (turning on)\n// fully re-derived. rank has no table -- it writes PageRank into frontmatter's own `_rank`\n// column -- so builder.ts handles it, and the links/rank pair, as special cases outside this map.\nexport const NARROW_FEATURE_TABLE: Readonly<Record<string, string>> = { tags: 'tags', sections: 'sections', links: 'links' };\n\n// links feeds rank's afterReconcile pass, so a links toggle always co-changes the `feature:rank`\n// segment too (featureEnabled's dependency); both are narrow-supported and builder.ts applies\n// them together. A changed set naming any other feature falls through to the existing full clear.\nconst NARROW_FEATURE_NAMES: ReadonlySet<string> = new Set([...Object.keys(NARROW_FEATURE_TABLE), 'rank']);\n\nexport interface FeatureToggle {\n  name: FeatureName;\n  // true: was off, now on (must fully re-derive). false: was on, now off (drop its rows).\n  turnedOn: boolean;\n}\n\n// Every changed key is a `feature:<name>` segment for a narrow-supported feature.\nexport function isFeatureOnlyChange(changedKeys: ReadonlySet<string>): boolean {\n  return changedKeys.size > 0 && [...changedKeys].every((key) => key.startsWith('feature:') && NARROW_FEATURE_NAMES.has(key.slice('feature:'.length)));\n}\n\n// One feature segment's old on/off, read back out of a stored `feature:<name>:on|off` segment.\n// Anything not in that exact shape returns null so the caller falls back to a full rebuild.\nfunction parseOnOff(sig: string, name: string): boolean | null {\n  const part = sig.split('|').find((p) => p.startsWith(`feature:${name}:`));\n  if (part === undefined) return null;\n  const fields = part.split(':');\n  if (fields.length !== 3 || (fields[2] !== 'on' && fields[2] !== 'off')) return null;\n  return fields[2] === 'on';\n}\n\n// Old-vs-new on/off for every changed feature key, or null when a segment can't be parsed back\n// out of `before` or `after`, or when it parses to the same state on both sides (should never\n// happen for a genuinely changed key) -- either way the caller takes the full rebuild.\nexport function classifyFeatureToggles(before: string, after: string, changedKeys: ReadonlySet<string>): FeatureToggle[] | null {\n  const toggles: FeatureToggle[] = [];\n  for (const key of changedKeys) {\n    const name = key.slice('feature:'.length) as FeatureName;\n    const wasOn = parseOnOff(before, name);\n    const isOn = parseOnOff(after, name);\n    if (wasOn === null || isOn === null || wasOn === isOn) return null;\n    toggles.push({ name, turnedOn: isOn });\n  }\n  return toggles;\n}\n"],"names":["NARROW_FEATURE_TABLE","tags","sections","links","NARROW_FEATURE_NAMES","Set","Object","keys","isFeatureOnlyChange","changedKeys","size","every","key","startsWith","has","slice","length","parseOnOff","sig","name","part","split","find","p","undefined","fields","classifyFeatureToggles","before","after","toggles","wasOn","isOn","push","turnedOn"],"mappings":"AAEA,+FAA+F;AAC/F,8EAA8E;AAE9E,6FAA6F;AAC7F,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,MAAMA,uBAAyD;IAAEC,MAAM;IAAQC,UAAU;IAAYC,OAAO;AAAQ,EAAE;AAE7H,iGAAiG;AACjG,8FAA8F;AAC9F,kGAAkG;AAClG,MAAMC,uBAA4C,IAAIC,IAAI;OAAIC,OAAOC,IAAI,CAACP;IAAuB;CAAO;AAQxG,kFAAkF;AAClF,OAAO,SAASQ,oBAAoBC,WAAgC;IAClE,OAAOA,YAAYC,IAAI,GAAG,KAAK;WAAID;KAAY,CAACE,KAAK,CAAC,CAACC,MAAQA,IAAIC,UAAU,CAAC,eAAeT,qBAAqBU,GAAG,CAACF,IAAIG,KAAK,CAAC,WAAWC,MAAM;AACnJ;AAEA,+FAA+F;AAC/F,4FAA4F;AAC5F,SAASC,WAAWC,GAAW,EAAEC,IAAY;IAC3C,MAAMC,OAAOF,IAAIG,KAAK,CAAC,KAAKC,IAAI,CAAC,CAACC,IAAMA,EAAEV,UAAU,CAAC,CAAC,QAAQ,EAAEM,KAAK,CAAC,CAAC;IACvE,IAAIC,SAASI,WAAW,OAAO;IAC/B,MAAMC,SAASL,KAAKC,KAAK,CAAC;IAC1B,IAAII,OAAOT,MAAM,KAAK,KAAMS,MAAM,CAAC,EAAE,KAAK,QAAQA,MAAM,CAAC,EAAE,KAAK,OAAQ,OAAO;IAC/E,OAAOA,MAAM,CAAC,EAAE,KAAK;AACvB;AAEA,+FAA+F;AAC/F,8FAA8F;AAC9F,uFAAuF;AACvF,OAAO,SAASC,uBAAuBC,MAAc,EAAEC,KAAa,EAAEnB,WAAgC;IACpG,MAAMoB,UAA2B,EAAE;IACnC,KAAK,MAAMjB,OAAOH,YAAa;QAC7B,MAAMU,OAAOP,IAAIG,KAAK,CAAC,WAAWC,MAAM;QACxC,MAAMc,QAAQb,WAAWU,QAAQR;QACjC,MAAMY,OAAOd,WAAWW,OAAOT;QAC/B,IAAIW,UAAU,QAAQC,SAAS,QAAQD,UAAUC,MAAM,OAAO;QAC9DF,QAAQG,IAAI,CAAC;YAAEb;YAAMc,UAAUF;QAAK;IACtC;IACA,OAAOF;AACT"}

package/dist/esm/store/index.d.ts

@@ -1,10 +1,15 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { OpenResult } from './sqlite/open.js';
+import type { Builder } from './builder.js';
+import type { OpenResult } from './open.js';
 export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & {
+    builder: Builder;
+}>;
 export declare function requireWatchConcurrency(cfg: ResolvedConfig): void;
 export { clearCache } from './cache.js';
+export type { OpenResult } from './open.js';
+export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
-export type { OpenResult } from './sqlite/open.js';
-export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
 export { CAPABILITY_NAMES } from './types.js';

package/dist/esm/store/index.js

@@ -2,26 +2,30 @@
 // beyond frontmatter + content lives in src/features/.
 import { anyPresetEmbeds, storeName } from '../config/index.js';
 import { SenseError } from '../errors.js';
-import { openDuckdb } from './duckdb/open.js';
+import { duckdbOpenDialect, openDuckdb } from './duckdb/open.js';
 import { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.js';
-import { openSqlite } from './sqlite/open.js';
+import { openWithBuilder } from './open.js';
+import { openSqlite, sqliteOpenDialect } from './sqlite/open.js';
 import { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.js';
-import { openTurso } from './turso/open.js';
+import { openTurso, tursoOpenDialect } from './turso/open.js';
 import { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.js';
 // Capabilities are checked against the registry entry before opening (no wasted connect/reconcile
 // work for a config that cannot be satisfied), then the store itself opens.
 const REGISTRY = {
     sqlite: {
         capabilities: SQLITE_CAPABILITIES,
-        open: openSqlite
+        open: openSqlite,
+        openForWatch: (cfg)=>openWithBuilder(cfg, sqliteOpenDialect)
     },
     duckdb: {
         capabilities: DUCKDB_CAPABILITIES,
-        open: openDuckdb
+        open: openDuckdb,
+        openForWatch: (cfg)=>openWithBuilder(cfg, duckdbOpenDialect)
     },
     turso: {
         capabilities: TURSO_CAPABILITIES,
-        open: openTurso
+        open: openTurso,
+        openForWatch: (cfg)=>openWithBuilder(cfg, tursoOpenDialect)
     }
 };
 function entryFor(name) {
@@ -39,6 +43,16 @@ export async function openStore(cfg) {
     }
     return entry.open(cfg);
 }
+// Same capability-checked open as openStore, but for a caller (only src/watch.ts) that must
+// reconcile again later: keeps the builder's pool warm and hands it back instead of closing it.
+export async function openStoreForWatch(cfg) {
+    const name = storeName(cfg);
+    const entry = entryFor(name);
+    if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
+        throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "vectors" in this build, but this config's "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)`);
+    }
+    return entry.openForWatch(cfg);
+}
 // watch keeps its connection open for the run, so a file-locking store would fail every
 // other command on the tree; check before open, as openStore does for "vectors".
 export function requireWatchConcurrency(cfg) {
@@ -48,6 +62,7 @@ export function requireWatchConcurrency(cfg) {
     }
 }
 export { clearCache } from './cache.js';
+export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
-export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export { CAPABILITY_NAMES } from './types.js';

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","clearCache","getMeta","setMeta","DB_FILENAME","docCount","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AACxC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,SAASC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB;AAEzE,SAASC,gBAAgB,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Builder } from './builder.ts';\nimport { duckdbOpenDialect, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './open.ts';\nimport { openWithBuilder } from './open.ts';\nimport { openSqlite, sqliteOpenDialect } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso, tursoOpenDialect } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n  // Same open, but keeps the builder's parse pool alive and hands it back: only a watcher, which\n  // reconciles repeatedly on the same connection, needs this.\n  openForWatch: (cfg: ResolvedConfig) => Promise<OpenResult & { builder: Builder }>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite, openForWatch: (cfg) => openWithBuilder(cfg, sqliteOpenDialect) },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb, openForWatch: (cfg) => openWithBuilder(cfg, duckdbOpenDialect) },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso, openForWatch: (cfg) => openWithBuilder(cfg, tursoOpenDialect) },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// Same capability-checked open as openStore, but for a caller (only src/watch.ts) that must\n// reconcile again later: keeps the builder's pool warm and hands it back instead of closing it.\nexport async function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & { builder: Builder }> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.openForWatch(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport type { OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","duckdbOpenDialect","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openWithBuilder","openSqlite","sqliteOpenDialect","SQLITE_CAPABILITIES","openTurso","tursoOpenDialect","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","openForWatch","cfg","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","has","openStoreForWatch","requireWatchConcurrency","clearCache","docCount","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,iBAAiB,EAAEC,UAAU,QAAQ,mBAAmB;AACjE,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,eAAe,QAAQ,YAAY;AAC5C,SAASC,UAAU,EAAEC,iBAAiB,QAAQ,mBAAmB;AACjE,SAASJ,gBAAgBK,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,kBAAkB;AAC9D,SAASP,gBAAgBQ,kBAAkB,QAAQ,mBAAmB;AAWtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcN;QAAqBO,MAAMT;QAAYU,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKV;IAAmB;IAC9HW,QAAQ;QAAEJ,cAAcV;QAAqBW,MAAMb;QAAYc,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKhB;IAAmB;IAC9HkB,OAAO;QAAEL,cAAcH;QAAoBI,MAAMN;QAAWO,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKP;IAAkB;AAC5H;AAEA,SAASU,SAASC,IAAe;IAC/B,MAAMC,QAAQV,QAAQ,CAACS,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAItB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEqB,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACZ,UAAUa,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUT,GAAmB;IACjD,MAAMI,OAAOtB,UAAUkB;IACvB,MAAMK,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIvB,gBAAgBmB,QAAQ,CAACK,MAAMR,YAAY,CAACa,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMP,IAAI,CAACE;AACpB;AAEA,4FAA4F;AAC5F,gGAAgG;AAChG,OAAO,eAAeW,kBAAkBX,GAAmB;IACzD,MAAMI,OAAOtB,UAAUkB;IACvB,MAAMK,QAAQF,SAASC;IAEvB,IAAIvB,gBAAgBmB,QAAQ,CAACK,MAAMR,YAAY,CAACa,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMN,YAAY,CAACC;AAC5B;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASY,wBAAwBZ,GAAmB;IACzD,MAAMI,OAAOtB,UAAUkB;IACvB,IAAI,CAACG,SAASC,MAAMP,YAAY,CAACa,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AAExC,SAASC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,WAAW,EAAEC,cAAc,QAAQ,mBAAmB;AAE/D,SAASC,gBAAgB,QAAQ,aAAa"}

package/dist/esm/store/open.d.ts

@@ -0,0 +1,15 @@
+import type { ResolvedConfig } from '../config/index.js';
+import type { Builder } from './builder.js';
+import type { OpenDialect, Store } from './types.js';
+export interface OpenResult {
+    store: Store;
+    cfg: ResolvedConfig;
+    dbPath: string;
+    parsed: number;
+    warnings: string[];
+}
+export declare function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult>;
+export declare function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & {
+    builder: Builder;
+}>;
+export declare function docCount(store: Store): Promise<number>;