sensemaking 0.24.6 → 0.25.0

package/dist/esm/store/duckdb/open.js

@@ -5,19 +5,23 @@ import { activeFeatures, FEATURES } from '../../features/index.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
+import { assertFtsReady, prepareFts } from './lexical.js';
 import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
 import { duckdbDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
+import { createVectorWriteStage } from './vectors.js';
 export const DB_FILENAME = 'cache.duckdb';
 // Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
 // The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
-export const SCHEMA_VERSION = '4';
-// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
+export const SCHEMA_VERSION = '5';
+// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,
+// while contains() verification reads the table directly.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
-async function ensureSchema(_handle, conn, cfg) {
+async function ensureSchema(handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
+    await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources ("path" TEXT PRIMARY KEY, text TEXT NOT NULL)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
     for (const feature of activeFeatures(cfg)){
@@ -25,6 +29,7 @@ async function ensureSchema(_handle, conn, cfg) {
         // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.
         if (feature.name === 'embed') {
             await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY ("path", chunk))`);
+            await createVectorWriteStage(handle.duckdb, STORE_DIMS);
             continue;
         }
         await feature.schema(conn);
@@ -37,7 +42,7 @@ async function close(handle) {
     handle.duckdb.disconnectSync();
     handle.instance.closeSync();
 }
-async function connect(dbPath, _cfg) {
+async function connect(dbPath, _cfg, options) {
     // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports
     // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.
     let DuckDBInstance;
@@ -52,7 +57,9 @@ async function connect(dbPath, _cfg) {
     try {
         // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns ("VARIANT
         // columns are not supported in storage versions prior to v1.5.0"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.
-        instance = await DuckDBInstance.create(dbPath, {
+        instance = await DuckDBInstance.create(dbPath, options.observational ? {
+            access_mode: 'READ_ONLY'
+        } : {
             storage_compatibility_version: 'v1.5.0'
         });
         duckdb = await instance.connect();
@@ -92,8 +99,10 @@ export const duckdbOpenDialect = {
     // "Cannot open file ... being used by another process". Both are the same condition.
     isLocked: (err)=>/Could not set lock on file|being used by another process/.test(err.message),
     ensureSchema,
+    prepareLexical: prepareFts,
+    assertLexicalReady: assertFtsReady,
     createStore: (handle, conn)=>createStore(handle.instance, handle.duckdb, conn)
 };
-export async function openDuckdb(cfg) {
-    return openWithDialect(cfg, duckdbOpenDialect);
+export async function openDuckdb(cfg, options) {
+    return openWithDialect(cfg, duckdbOpenDialect, options);
 }

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","assertFtsReady","prepareFts","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureSchema","handle","conn","cfg","exec","feature","name","duckdb","schema","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","options","DuckDBInstance","err","message","create","observational","access_mode","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","prepareLexical","assertLexicalReady","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,UAAU,QAAQ,eAAe;AAC1D,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,eAAeC,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;IAC7E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWrB,eAAemB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEtB,WAAW,+BAA+B,CAAC;YAChM,MAAMe,uBAAuBI,OAAOM,MAAM,EAAEzB;YAC5C;QACF;QACA,MAAMuB,QAAQG,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMf,QAAQe,MAAM,sBAAuB,MAAM,MAAMd,QAAQc,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMZ,QAAQe,MAAM,gBAAiB,MAAM,MAAMd,QAAQc,MAAM,YAAYrB,iBAAiBsB,KAAKlB;AACxG;AAEA,iFAAiF;AACjF,eAAewB,MAAMR,MAAoB;IACvCA,OAAOM,MAAM,CAACG,cAAc;IAC5BT,OAAOU,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB,EAAEC,OAA0D;IACrH,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMxB,WAAU;IACxC,EAAE,OAAOyB,KAAK;QACZ,IAAIA,eAAenC,YAAY,MAAMmC;QACrC,MAAM,IAAInC,WAAW,4BAA4B,CAAC,yBAAyB,EAAES,eAAe,UAAU,EAAE,AAAC0B,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIZ;IACJ,IAAII;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMM,eAAeG,MAAM,CAACN,QAAQE,QAAQK,aAAa,GAAG;YAAEC,aAAa;QAAY,IAAI;YAAEC,+BAA+B;QAAS;QAChJhB,SAAS,MAAMI,SAASE,OAAO;IACjC,EAAE,OAAOK,KAAK;QACZ,wFAAwF;QACxFP,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI7B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE+B,OAAO,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMxB,kBAAkBY;QACxB,MAAML,OAAOb,iBAAiBkB;QAC9B,OAAO;YAAEN,QAAQ;gBAAEU;gBAAUJ;YAAO;YAAGL;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMT,MAAM;YAAEE;YAAUJ;QAAO;QAC/B,MAAMW;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMM,oBAA+C;IAC1DC,UAAU3B;IACV4B,eAAe3B;IACf4B,kBAAkBjC;IAClBmB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFmB,UAAU,CAACV,MAAQ,2DAA2DW,IAAI,CAACX,IAAIC,OAAO;IAC9FnB;IACA8B,gBAAgBvC;IAChBwC,oBAAoBzC;IACpBM,aAAa,CAACK,QAAQC,OAASN,YAAYK,OAAOU,QAAQ,EAAEV,OAAOM,MAAM,EAAEL;AAC7E,EAAE;AAEF,OAAO,eAAe8B,WAAW7B,GAAmB,EAAEa,OAA6B;IACjF,OAAO9B,gBAAgBiB,KAAKqB,mBAAmBR;AACjD"}

package/dist/esm/store/duckdb/reconcile.js

@@ -32,8 +32,8 @@ async function reconcileContent(conn, touched, docs, _delta) {
     // The delete above cleared every touched path, and duckdb holds the cache file for its whole
     // connection, so no second writer can have landed one of these rows: nothing can conflict.
     await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));
-    // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
-    // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.
+    // Build prepares the FTS index after reconcile. This persisted mark commits with the content
+    // change, so an interrupted build cannot expose stale lexical results.
     await markContentStale(conn);
 }
 // DuckDB rejects more than one ALTER command per statement ("Parser Error: Only one ALTER

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","appendRows","quoteIdent","markContentStale","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","addColumns","names","exec","name","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AAEtD,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,MAAMC,qBAAqB,CAAC,qBAAqB,EAAED,gBAAgBE,GAAG,CAACJ,YAAYK,IAAI,CAAC,MAAM,qBAAqB,CAAC;AAEpH,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQZ,GAAG,CAAC,CAACiB,IAAM;YAACA;SAAE;IAE1B,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAMtB,WAAWgB,MAAM,WAAWb,iBAAiBC,oBAAoBc,KAAKb,GAAG,CAACE;IAChF,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAML,iBAAiBc;AACzB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeO,WAAWP,IAAgB,EAAEQ,KAAe;IACzD,IAAIA,MAAMJ,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKS,IAAI,CAACD,MAAMnB,GAAG,CAAC,CAACqB,OAAS,CAAC,mCAAmC,EAAEzB,WAAWyB,MAAM,QAAQ,CAAC,EAAEpB,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMqB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhB,yBAAyB;YACnC,MAAM,IAAIf,WAAW,gBAAgB,CAAC,uBAAuB,EAAE+B,MAAM,8CAA8C,EAAEhB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAS;IACAR;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // Build prepares the FTS index after reconcile. This persisted mark commits with the content\n  // change, so an interrupted build cannot expose stale lexical results.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","appendRows","quoteIdent","markContentStale","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","addColumns","names","exec","name","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AAEtD,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,MAAMC,qBAAqB,CAAC,qBAAqB,EAAED,gBAAgBE,GAAG,CAACJ,YAAYK,IAAI,CAAC,MAAM,qBAAqB,CAAC;AAEpH,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQZ,GAAG,CAAC,CAACiB,IAAM;YAACA;SAAE;IAE1B,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAMtB,WAAWgB,MAAM,WAAWb,iBAAiBC,oBAAoBc,KAAKb,GAAG,CAACE;IAChF,6FAA6F;IAC7F,uEAAuE;IACvE,MAAML,iBAAiBc;AACzB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeO,WAAWP,IAAgB,EAAEQ,KAAe;IACzD,IAAIA,MAAMJ,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKS,IAAI,CAACD,MAAMnB,GAAG,CAAC,CAACqB,OAAS,CAAC,mCAAmC,EAAEzB,WAAWyB,MAAM,QAAQ,CAAC,EAAEpB,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMqB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhB,yBAAyB;YACnC,MAAM,IAAIf,WAAW,gBAAgB,CAAC,uBAAuB,EAAE+B,MAAM,8CAA8C,EAAEhB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAS;IACAR;AACF,EAAE"}

package/dist/esm/store/duckdb/vectors.d.ts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/esm/store/duckdb/vectors.js

@@ -1,6 +1,5 @@
 import { withTransaction } from '../transaction.js';
 import { asCosine, sampleEvenly } from '../vectors.js';
-import { rewriteUpdate } from './batch.js';
 import { duckdbApi } from './native.js';
 // This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and
 // scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.
@@ -9,6 +8,11 @@ import { duckdbApi } from './native.js';
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 const untyped = undefined;
+const VECTOR_WRITE_STAGE = '_sense_vector_write_stage';
+// Created once per native connection outside write transactions; its lifetime follows that connection.
+export async function createVectorWriteStage(duckdb, dims) {
+    await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} ("path" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);
+}
 // The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);
 // zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.
 function padded(values, dims) {
@@ -27,27 +31,27 @@ function dequantize(row, dims) {
     const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
     return padded(Array.from(q, (v)=>v * row.scale), dims);
 }
-// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the
-// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
+// The native appender needs an explicit FLOAT array type: inference from an integer first component
+// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.
 export async function writeVectorBatch(duckdb, conn, dims, rows) {
     if (rows.length === 0) return;
-    const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
-    const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE "path" = ? AND chunk = ?', rows.length);
-    if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');
+    const { ARRAY, FLOAT } = await duckdbApi();
+    const vectorType = ARRAY(FLOAT, dims);
     await withTransaction(conn, async ()=>{
-        const stmt = await duckdb.prepare(rewritten.sql);
+        await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);
+        const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);
         try {
-            const values = [];
-            const types = [];
             for (const row of rows){
-                values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);
-                types.push(ARRAY(DOUBLE, dims), untyped, untyped);
+                appender.appendVarchar(row.path);
+                appender.appendInteger(row.chunk);
+                appender.appendArray(dequantize(row, dims), vectorType);
+                appender.endRow();
             }
-            stmt.bind(values, types);
-            await stmt.run();
+            appender.flushSync();
         } finally{
-            stmt.destroySync();
+            appender.closeSync();
         }
+        await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings."path" = staged."path" AND embeddings.chunk = staged.chunk`);
     });
 }
 // Best chunk per file by cosine, with the earliest chunk winning exact score ties.

package/dist/esm/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","query","queryNorm","sqrt","reduce","sum","value","score","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,oIAAoI;AACpI,8QAA8Q;AAC9Q,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,mFAAmF;AACnF,oGAAoG;AACpG,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMG,QAAQlD,OAAO6C,IAAI3C;IACzB,MAAMiD,YAAY5C,KAAK6C,IAAI,CAACF,MAAMG,MAAM,CAAC,CAACC,KAAKC,QAAUD,MAAMC,QAAQA,OAAO;IAC9E,MAAMC,QAAQL,cAAc,IAAI,QAAQ;IACxC,MAAMf,MAAM,CAAC;;;;;oDAKqC,EAAEoB,MAAM;;iCAE3B,EAAET,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKvF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAKkD,cAAc,IAAI,EAAE,GAAG;gBAACtB,WAAWqB;aAAO;eAAMD;YAAaH;SAAM;QACtG,MAAMT,QAAsB;eAAKc,cAAc,IAAI,EAAE,GAAG;gBAACrB,MAAMC,QAAQ7B;aAAM;eAAM+C,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC3HoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAEuB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACjH,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeuB,YAAYxC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAE4B,IAAgE;IACxK,IAAIA,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMoB,aAAa,MAAMzC,KAAKQ,OAAO,CAAC;IACtC,MAAMkC,aAAc,MAAMD,WAAWE,GAAG,CAAC/B;IACzC,IAAI8B,WAAW5D,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAM8D,UAAU5E,aAAa0E,YAAYZ,GAAG,CAAC,CAACzC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAM2E,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMxB,cAAckB,KAAKpB,OAAO,GAAG;WAAIoB,KAAKpB,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAEmC,QAAQd,GAAG,CAAC,IAAM,OAAO3C,IAAI,CAAC,MAAM;;;;MAI3E,EAAE0D,YAAY/D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAY8D,YAAY/D,MAAM,GAAG,GAAG,GAAG;MACtF,EAAE0D,KAAKpB,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIsE,QAAQd,GAAG,CAAC,CAACiB,IAAM7C,WAAW7B,OAAO0E,GAAGxE;YAASqC;eAASiC;eAAgBvB;YAAakB,KAAKQ,CAAC;SAAC;QAChI,MAAMtC,QAAsB;eAAIkC,QAAQd,GAAG,CAAC,IAAM3B,MAAMC,QAAQ7B;YAAQJ;eAAY0E,YAAYf,GAAG,CAAC,IAAM3D;eAAamD,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAE0B,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACxE,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\nconst VECTOR_WRITE_STAGE = '_sense_vector_write_stage';\n\n// Created once per native connection outside write transactions; its lifetime follows that connection.\nexport async function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void> {\n  await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} (\"path\" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// The native appender needs an explicit FLOAT array type: inference from an integer first component\n// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { ARRAY, FLOAT } = await duckdbApi();\n  const vectorType = ARRAY(FLOAT, dims);\n\n  await withTransaction(conn, async () => {\n    await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);\n    const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);\n    try {\n      for (const row of rows) {\n        appender.appendVarchar(row.path);\n        appender.appendInteger(row.chunk);\n        appender.appendArray(dequantize(row, dims), vectorType);\n        appender.endRow();\n      }\n      appender.flushSync();\n    } finally {\n      appender.closeSync();\n    }\n    await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings.\"path\" = staged.\"path\" AND embeddings.chunk = staged.chunk`);\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","duckdbApi","untyped","undefined","VECTOR_WRITE_STAGE","createVectorWriteStage","duckdb","dims","run","padded","values","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","conn","rows","ARRAY","FLOAT","vectorType","appender","createAppender","appendVarchar","path","appendInteger","chunk","appendArray","endRow","flushSync","closeSync","scanCandidates","qv","fetch","allowed","size","arrayValue","DOUBLE","allowedList","query","queryNorm","sqrt","reduce","sum","value","score","sql","stmt","prepare","types","map","bind","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","destroySync","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAChB,MAAMC,qBAAqB;AAE3B,uGAAuG;AACvG,OAAO,eAAeC,uBAAuBC,MAAwB,EAAEC,IAAY;IACjF,MAAMD,OAAOE,GAAG,CAAC,CAAC,kBAAkB,EAAEJ,mBAAmB,2CAA2C,EAAEG,KAAK,EAAE,CAAC;AAChH;AAEA,wHAAwH;AACxH,oIAAoI;AACpI,SAASE,OAAOC,MAAyB,EAAEH,IAAY;IACrD,MAAMI,MAAM,IAAIC,MAAcL,MAAMM,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEV,OAAOO,IAAKH,GAAG,CAACG,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,MAAMkB,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOrB,OACLG,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCzB;AAEJ;AAEA,oGAAoG;AACpG,gHAAgH;AAChH,OAAO,eAAe0B,iBAAiB3B,MAAwB,EAAE4B,IAAgB,EAAE3B,IAAY,EAAE4B,IAAsB;IACrH,IAAIA,KAAKlB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEmB,KAAK,EAAEC,KAAK,EAAE,GAAG,MAAMpC;IAC/B,MAAMqC,aAAaF,MAAMC,OAAO9B;IAEhC,MAAMT,gBAAgBoC,MAAM;QAC1B,MAAM5B,OAAOE,GAAG,CAAC,CAAC,YAAY,EAAEJ,oBAAoB;QACpD,MAAMmC,WAAW,MAAMjC,OAAOkC,cAAc,CAACpC;QAC7C,IAAI;YACF,KAAK,MAAMoB,OAAOW,KAAM;gBACtBI,SAASE,aAAa,CAACjB,IAAIkB,IAAI;gBAC/BH,SAASI,aAAa,CAACnB,IAAIoB,KAAK;gBAChCL,SAASM,WAAW,CAACtB,WAAWC,KAAKjB,OAAO+B;gBAC5CC,SAASO,MAAM;YACjB;YACAP,SAASQ,SAAS;QACpB,SAAU;YACRR,SAASS,SAAS;QACpB;QACA,MAAM1C,OAAOE,GAAG,CAAC,CAAC,kDAAkD,EAAEJ,mBAAmB,sFAAsF,CAAC;IAClL;AACF;AAEA,mFAAmF;AACnF,oGAAoG;AACpG,OAAO,eAAe6C,eAAe3C,MAAwB,EAAE4C,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEC,UAAU,EAAElB,KAAK,EAAEmB,MAAM,EAAE,GAAG,MAAMtD;IAC5C,MAAMuD,cAAcJ,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMK,QAAQhD,OAAOyC,IAAI3C;IACzB,MAAMmD,YAAY3C,KAAK4C,IAAI,CAACF,MAAMG,MAAM,CAAC,CAACC,KAAKC,QAAUD,MAAMC,QAAQA,OAAO;IAC9E,MAAMC,QAAQL,cAAc,IAAI,QAAQ;IACxC,MAAMM,MAAM,CAAC;;;;;oDAKqC,EAAED,MAAM;;iCAE3B,EAAEX,UAAU,CAAC,IAAI,EAAElC,SAAS,UAAUsC,YAAYvC,MAAM,GAAG,GAAG,GAAG;;;;;WAKvF,CAAC;IACV,MAAMgD,OAAO,MAAM3D,OAAO4D,OAAO,CAACF;IAClC,IAAI;QACF,MAAMtD,SAAwB;eAAKgD,cAAc,IAAI,EAAE,GAAG;gBAACJ,WAAWG;aAAO;eAAMD;YAAaL;SAAM;QACtG,MAAMgB,QAAsB;eAAKT,cAAc,IAAI,EAAE,GAAG;gBAACtB,MAAMmB,QAAQhD;aAAM;eAAMiD,YAAYY,GAAG,CAAC,IAAMlE;YAAUA;SAAQ;QAC3H+D,KAAKI,IAAI,CAAC3D,QAAQyD;QAClB,MAAMG,SAAS,MAAML,KAAKM,aAAa;QACvC,MAAMpC,OAAOmC,OAAOE,eAAe;QACnC,OAAOrC,KAAKiC,GAAG,CAAC,CAACK,IAAO,CAAA;gBAAE/B,MAAM+B,EAAE/B,IAAI;gBAAEgC,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAY9E,SAAS0E,EAAEV,KAAK;YAAE,CAAA;IACjH,SAAU;QACRE,KAAKa,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeC,YAAYzE,MAAwB,EAAE4B,IAAgB,EAAE3B,IAAY,EAAEmC,IAAY,EAAEsC,IAAgE;IACxK,IAAIA,KAAK5B,OAAO,IAAI4B,KAAK5B,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAM4B,aAAa,MAAM/C,KAAKgC,OAAO,CAAC;IACtC,MAAMgB,aAAc,MAAMD,WAAWE,GAAG,CAACzC;IACzC,IAAIwC,WAAWjE,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMmE,UAAUpF,aAAakF,YAAYd,GAAG,CAAC,CAAC5C,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAE2B,UAAU,EAAElB,KAAK,EAAEmB,MAAM,EAAE,GAAG,MAAMtD;IAC5C,MAAMoF,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAM9B,cAAcwB,KAAK5B,OAAO,GAAG;WAAI4B,KAAK5B,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMY,MAAM,CAAC,4BAA4B,EAAEoB,QAAQhB,GAAG,CAAC,IAAM,OAAO9C,IAAI,CAAC,MAAM;;;;MAI3E,EAAE+D,YAAYpE,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYmE,YAAYpE,MAAM,GAAG,GAAG,GAAG;MACtF,EAAE+D,KAAK5B,OAAO,GAAG,CAAC,IAAI,EAAElC,SAAS,YAAYsC,YAAYvC,MAAM,GAAG,GAAG,GAAG;;;;;WAKnE,CAAC;IACV,MAAMgD,OAAO,MAAM3D,OAAO4D,OAAO,CAACF;IAClC,IAAI;QACF,MAAMtD,SAAwB;eAAI0E,QAAQhB,GAAG,CAAC,CAACmB,IAAMjC,WAAW7C,OAAO8E,GAAGhF;YAASmC;eAAS2C;eAAgB7B;YAAawB,KAAKQ,CAAC;SAAC;QAChI,MAAMrB,QAAsB;eAAIiB,QAAQhB,GAAG,CAAC,IAAMhC,MAAMmB,QAAQhD;YAAQL;eAAYmF,YAAYjB,GAAG,CAAC,IAAMlE;eAAasD,YAAYY,GAAG,CAAC,IAAMlE;YAAUA;SAAQ;QAC/J+D,KAAKI,IAAI,CAAC3D,QAAQyD;QAClB,MAAMG,SAAS,MAAML,KAAKM,aAAa;QACvC,MAAMpC,OAAOmC,OAAOE,eAAe;QACnC,OAAOrC,KAAKiC,GAAG,CAAC,CAACK,IAAO,CAAA;gBAAE/B,MAAM+B,EAAE/B,IAAI;gBAAEmC,YAAY9E,SAAS0E,EAAEV,KAAK;YAAE,CAAA;IACxE,SAAU;QACRE,KAAKa,WAAW;IAClB;AACF"}

package/dist/esm/store/index.d.ts

@@ -1,11 +1,30 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { OpenResult } from './open.js';
-export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
-export { clearCache } from './cache.js';
-export type { OpenResult } from './open.js';
+import { clearCache } from './cache.js';
+import type { InternalOpenOptions, OpenOptions, OpenResult } from './open.js';
+import type { Stages } from './stages.js';
+export declare function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult>;
+export declare function openStore(cfg: ResolvedConfig, options?: OpenOptions): Promise<OpenResult>;
+export declare function lexicalReadiness(cfg: ResolvedConfig): Promise<{
+    ready: boolean;
+    error: string | null;
+}>;
+export interface BuildOptions {
+    force?: boolean;
+}
+export interface BuildResult {
+    cfg: ResolvedConfig;
+    dbPath: string;
+    parsed: number;
+    warnings: string[];
+    stages: Stages;
+}
+export declare function buildIndex(cfg: ResolvedConfig, options?: BuildOptions): Promise<BuildResult>;
+export declare function storeFilename(cfg: ResolvedConfig): string;
+export type { OpenOptions, OpenResult } from './open.js';
 export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
 export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Stages } from './stages.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
 export { CAPABILITY_NAMES } from './types.js';
+export { clearCache };

package/dist/esm/store/index.js

@@ -1,26 +1,28 @@
-// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything
-// beyond frontmatter + content lives in src/features/.
+// Rows -> a backing store: registry + the public open/build boundary. Parsing lives in scan.ts;
+// everything beyond frontmatter + content lives in src/features/.
 import { anyPresetEmbeds, storeName } from '../config/index.js';
 import { SenseError } from '../errors.js';
-import { openDuckdb } from './duckdb/open.js';
+import { clearCache } from './cache.js';
+import { DB_FILENAME as DUCKDB_FILENAME, openDuckdb } from './duckdb/open.js';
 import { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.js';
-import { openSqlite } from './sqlite/open.js';
+import { openSqlite, DB_FILENAME as SQLITE_FILENAME } from './sqlite/open.js';
 import { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.js';
-import { openTurso } from './turso/open.js';
+import { openTurso, DB_FILENAME as TURSO_FILENAME } from './turso/open.js';
 import { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.js';
-// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile
-// work for a config that cannot be satisfied), then the store itself opens.
 const REGISTRY = {
     sqlite: {
         capabilities: SQLITE_CAPABILITIES,
+        filename: SQLITE_FILENAME,
         open: openSqlite
     },
     duckdb: {
         capabilities: DUCKDB_CAPABILITIES,
+        filename: DUCKDB_FILENAME,
         open: openDuckdb
     },
     turso: {
         capabilities: TURSO_CAPABILITIES,
+        filename: TURSO_FILENAME,
         open: openTurso
     }
 };
@@ -29,18 +31,81 @@ function entryFor(name) {
     if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store "${name}"; available: ${Object.keys(REGISTRY).join(', ')}`);
     return entry;
 }
-export async function openStore(cfg) {
-    const name = storeName(cfg);
-    const entry = entryFor(name);
-    // The one unambiguous, config-level capability need: an embed block some preset actually uses
-    // for vectors. Lexical search has no such signal at this level (every preset defaults to "words"), so a store lacking it fails loudly at first use.
-    if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
+function requirementsFor(cfg) {
+    const requirements = new Set([
+        'core',
+        'lexical'
+    ]);
+    if (anyPresetEmbeds(cfg)) requirements.add('vectors');
+    return requirements;
+}
+function validateRequirements(cfg, entry, requirements) {
+    if (requirements.has('vectors') && anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
+        const name = storeName(cfg);
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "vectors" in this build, but this config's "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)`);
     }
-    return entry.open(cfg);
 }
-export { clearCache } from './cache.js';
+export async function openStoreFor(cfg, options) {
+    var _options_requirements;
+    const entry = entryFor(storeName(cfg));
+    validateRequirements(cfg, entry, (_options_requirements = options.requirements) !== null && _options_requirements !== void 0 ? _options_requirements : new Set([
+        'core'
+    ]));
+    return entry.open(cfg, options);
+}
+export async function openStore(cfg, options = {}) {
+    const buildEnabled = options.build !== false;
+    return openStoreFor(cfg, {
+        build: buildEnabled,
+        requirements: buildEnabled ? requirementsFor(cfg) : new Set([
+            'core'
+        ])
+    });
+}
+export async function lexicalReadiness(cfg) {
+    let opened;
+    try {
+        opened = await openStoreFor(cfg, {
+            build: false,
+            requirements: new Set([
+                'core',
+                'lexical'
+            ])
+        });
+    } catch (err) {
+        if (err instanceof SenseError && err.code === 'INDEX_NOT_READY') return {
+            ready: false,
+            error: err.message
+        };
+        throw err;
+    }
+    try {
+        return {
+            ready: true,
+            error: null
+        };
+    } finally{
+        await opened.store.close();
+    }
+}
+export async function buildIndex(cfg, options = {}) {
+    if (options.force) clearCache(cfg);
+    const opened = await openStoreFor(cfg, {
+        build: true,
+        requirements: requirementsFor(cfg)
+    });
+    try {
+        const { store: _store, ...result } = opened;
+        return result;
+    } finally{
+        await opened.store.close();
+    }
+}
+export function storeFilename(cfg) {
+    return entryFor(storeName(cfg)).filename;
+}
 export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
 export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export { CAPABILITY_NAMES } from './types.js';
+export { clearCache };

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\nexport { clearCache } from './cache.ts';\nexport type { OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Stages } from './stages.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","clearCache","docCount","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,SAASE,UAAU,QAAQ,aAAa;AAExC,SAASC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,WAAW,EAAEC,cAAc,QAAQ,mBAAmB;AAG/D,SAASC,gBAAgB,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + the public open/build boundary. Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { clearCache } from './cache.ts';\nimport { DB_FILENAME as DUCKDB_FILENAME, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { BuildRequirement, InternalOpenOptions, OpenOptions, OpenResult } from './open.ts';\nimport { openSqlite, DB_FILENAME as SQLITE_FILENAME } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Stages } from './stages.ts';\nimport { openTurso, DB_FILENAME as TURSO_FILENAME } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  filename: string;\n  open: (cfg: ResolvedConfig, options?: InternalOpenOptions) => Promise<OpenResult>;\n}\n\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, filename: SQLITE_FILENAME, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, filename: DUCKDB_FILENAME, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, filename: TURSO_FILENAME, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nfunction requirementsFor(cfg: ResolvedConfig): Set<BuildRequirement> {\n  const requirements = new Set<BuildRequirement>(['core', 'lexical']);\n  if (anyPresetEmbeds(cfg)) requirements.add('vectors');\n  return requirements;\n}\n\nfunction validateRequirements(cfg: ResolvedConfig, entry: StoreEntry, requirements: ReadonlySet<BuildRequirement>): void {\n  if (requirements.has('vectors') && anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    const name = storeName(cfg);\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n}\n\nexport async function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult> {\n  const entry = entryFor(storeName(cfg));\n  validateRequirements(cfg, entry, options.requirements ?? new Set<BuildRequirement>(['core']));\n  return entry.open(cfg, options);\n}\n\nexport async function openStore(cfg: ResolvedConfig, options: OpenOptions = {}): Promise<OpenResult> {\n  const buildEnabled = options.build !== false;\n  return openStoreFor(cfg, {\n    build: buildEnabled,\n    requirements: buildEnabled ? requirementsFor(cfg) : new Set<BuildRequirement>(['core']),\n  });\n}\n\nexport async function lexicalReadiness(cfg: ResolvedConfig): Promise<{ ready: boolean; error: string | null }> {\n  let opened: OpenResult;\n  try {\n    opened = await openStoreFor(cfg, { build: false, requirements: new Set<BuildRequirement>(['core', 'lexical']) });\n  } catch (err) {\n    if (err instanceof SenseError && err.code === 'INDEX_NOT_READY') return { ready: false, error: err.message };\n    throw err;\n  }\n  try {\n    return { ready: true, error: null };\n  } finally {\n    await opened.store.close();\n  }\n}\n\nexport interface BuildOptions {\n  force?: boolean;\n}\n\nexport interface BuildResult {\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\nexport async function buildIndex(cfg: ResolvedConfig, options: BuildOptions = {}): Promise<BuildResult> {\n  if (options.force) clearCache(cfg);\n  const opened = await openStoreFor(cfg, { build: true, requirements: requirementsFor(cfg) });\n  try {\n    const { store: _store, ...result } = opened;\n    return result;\n  } finally {\n    await opened.store.close();\n  }\n}\n\nexport function storeFilename(cfg: ResolvedConfig): string {\n  return entryFor(storeName(cfg)).filename;\n}\n\nexport type { OpenOptions, OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Stages } from './stages.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\nexport { clearCache };\n"],"names":["anyPresetEmbeds","storeName","SenseError","clearCache","DB_FILENAME","DUCKDB_FILENAME","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_FILENAME","SQLITE_CAPABILITIES","openTurso","TURSO_FILENAME","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","filename","open","duckdb","turso","entryFor","name","entry","Object","keys","join","requirementsFor","cfg","requirements","Set","add","validateRequirements","has","openStoreFor","options","openStore","buildEnabled","build","lexicalReadiness","opened","err","code","ready","error","message","store","close","buildIndex","force","_store","result","storeFilename","docCount","getMeta","setMeta","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,gGAAgG;AAChG,kEAAkE;AAGlE,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,aAAa;AACxC,SAASC,eAAeC,eAAe,EAAEC,UAAU,QAAQ,mBAAmB;AAC9E,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,EAAEL,eAAeM,eAAe,QAAQ,mBAAmB;AAC9E,SAASH,gBAAgBI,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,SAAS,EAAER,eAAeS,cAAc,QAAQ,kBAAkB;AAC3E,SAASN,gBAAgBO,kBAAkB,QAAQ,mBAAmB;AAStE,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcN;QAAqBO,UAAUR;QAAiBS,MAAMV;IAAW;IACzFW,QAAQ;QAAEH,cAAcT;QAAqBU,UAAUb;QAAiBc,MAAMb;IAAW;IACzFe,OAAO;QAAEJ,cAAcH;QAAoBI,UAAUL;QAAgBM,MAAMP;IAAU;AACvF;AAEA,SAASU,SAASC,IAAe;IAC/B,MAAMC,QAAQT,QAAQ,CAACQ,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAItB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEqB,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACX,UAAUY,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,SAASI,gBAAgBC,GAAmB;IAC1C,MAAMC,eAAe,IAAIC,IAAsB;QAAC;QAAQ;KAAU;IAClE,IAAI/B,gBAAgB6B,MAAMC,aAAaE,GAAG,CAAC;IAC3C,OAAOF;AACT;AAEA,SAASG,qBAAqBJ,GAAmB,EAAEL,KAAiB,EAAEM,YAA2C;IAC/G,IAAIA,aAAaI,GAAG,CAAC,cAAclC,gBAAgB6B,QAAQ,CAACL,MAAMP,YAAY,CAACiB,GAAG,CAAC,YAAY;QAC7F,MAAMX,OAAOtB,UAAU4B;QACvB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;AACF;AAEA,OAAO,eAAeY,aAAaN,GAAmB,EAAEO,OAA4B;QAEjDA;IADjC,MAAMZ,QAAQF,SAASrB,UAAU4B;IACjCI,qBAAqBJ,KAAKL,QAAOY,wBAAAA,QAAQN,YAAY,cAApBM,mCAAAA,wBAAwB,IAAIL,IAAsB;QAAC;KAAO;IAC3F,OAAOP,MAAML,IAAI,CAACU,KAAKO;AACzB;AAEA,OAAO,eAAeC,UAAUR,GAAmB,EAAEO,UAAuB,CAAC,CAAC;IAC5E,MAAME,eAAeF,QAAQG,KAAK,KAAK;IACvC,OAAOJ,aAAaN,KAAK;QACvBU,OAAOD;QACPR,cAAcQ,eAAeV,gBAAgBC,OAAO,IAAIE,IAAsB;YAAC;SAAO;IACxF;AACF;AAEA,OAAO,eAAeS,iBAAiBX,GAAmB;IACxD,IAAIY;IACJ,IAAI;QACFA,SAAS,MAAMN,aAAaN,KAAK;YAAEU,OAAO;YAAOT,cAAc,IAAIC,IAAsB;gBAAC;gBAAQ;aAAU;QAAE;IAChH,EAAE,OAAOW,KAAK;QACZ,IAAIA,eAAexC,cAAcwC,IAAIC,IAAI,KAAK,mBAAmB,OAAO;YAAEC,OAAO;YAAOC,OAAOH,IAAII,OAAO;QAAC;QAC3G,MAAMJ;IACR;IACA,IAAI;QACF,OAAO;YAAEE,OAAO;YAAMC,OAAO;QAAK;IACpC,SAAU;QACR,MAAMJ,OAAOM,KAAK,CAACC,KAAK;IAC1B;AACF;AAcA,OAAO,eAAeC,WAAWpB,GAAmB,EAAEO,UAAwB,CAAC,CAAC;IAC9E,IAAIA,QAAQc,KAAK,EAAE/C,WAAW0B;IAC9B,MAAMY,SAAS,MAAMN,aAAaN,KAAK;QAAEU,OAAO;QAAMT,cAAcF,gBAAgBC;IAAK;IACzF,IAAI;QACF,MAAM,EAAEkB,OAAOI,MAAM,EAAE,GAAGC,QAAQ,GAAGX;QACrC,OAAOW;IACT,SAAU;QACR,MAAMX,OAAOM,KAAK,CAACC,KAAK;IAC1B;AACF;AAEA,OAAO,SAASK,cAAcxB,GAAmB;IAC/C,OAAOP,SAASrB,UAAU4B,MAAMX,QAAQ;AAC1C;AAGA,SAASoC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASpD,WAAW,EAAEqD,cAAc,QAAQ,mBAAmB;AAG/D,SAASC,gBAAgB,QAAQ,aAAa;AAC9C,SAASvD,UAAU,GAAG"}

package/dist/esm/store/open.d.ts

@@ -1,6 +1,13 @@
 import type { ResolvedConfig } from '../config/index.js';
 import type { Stages } from './stages.js';
 import type { OpenDialect, Store } from './types.js';
+export type BuildRequirement = 'core' | 'lexical' | 'vectors';
+export interface OpenOptions {
+    build?: boolean;
+}
+export interface InternalOpenOptions extends OpenOptions {
+    requirements?: ReadonlySet<BuildRequirement>;
+}
 export interface OpenResult {
     store: Store;
     cfg: ResolvedConfig;
@@ -9,5 +16,6 @@ export interface OpenResult {
     warnings: string[];
     stages: Stages;
 }
-export declare function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult>;
+export declare function prepareDocumentEmbeddings(store: Store, cfg: ResolvedConfig, allowed?: ReadonlySet<string>): Promise<void>;
+export declare function openWithDialect<Handle>(input: ResolvedConfig, dialect: OpenDialect<Handle>, options?: InternalOpenOptions): Promise<OpenResult>;
 export declare function docCount(store: Store): Promise<number>;