sensemaking 0.24.7 → 0.25.0

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureSchema","handle","conn","cfg","exec","feature","name","duckdb","schema","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;IAC7E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWnB,eAAeiB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEpB,WAAW,+BAA+B,CAAC;YAChM,MAAMa,uBAAuBI,OAAOM,MAAM,EAAEvB;YAC5C;QACF;QACA,MAAMqB,QAAQG,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,iFAAiF;AACjF,eAAesB,MAAMR,MAAoB;IACvCA,OAAOM,MAAM,CAACG,cAAc;IAC5BT,OAAOU,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIX;IACJ,IAAII;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFb,SAAS,MAAMI,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBY;QACxB,MAAML,OAAOX,iBAAiBgB;QAC9B,OAAO;YAAEN,QAAQ;gBAAEU;gBAAUJ;YAAO;YAAGL;QAAK;IAC9C,EAAE,OAAOe,KAAK;QACZ,MAAMR,MAAM;YAAEE;YAAUJ;QAAO;QAC/B,MAAMU;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUxB;IACVyB,eAAexB;IACfyB,kBAAkB9B;IAClBmB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFgB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FlB;IACAJ,aAAa,CAACK,QAAQC,OAASN,YAAYK,OAAOU,QAAQ,EAAEV,OAAOM,MAAM,EAAEL;AAC7E,EAAE;AAEF,OAAO,eAAeyB,WAAWxB,GAAmB;IAClD,OAAOf,gBAAgBe,KAAKkB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","assertFtsReady","prepareFts","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureSchema","handle","conn","cfg","exec","feature","name","duckdb","schema","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","options","DuckDBInstance","err","message","create","observational","access_mode","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","prepareLexical","assertLexicalReady","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,UAAU,QAAQ,eAAe;AAC1D,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,eAAeC,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;IAC7E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWrB,eAAemB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEtB,WAAW,+BAA+B,CAAC;YAChM,MAAMe,uBAAuBI,OAAOM,MAAM,EAAEzB;YAC5C;QACF;QACA,MAAMuB,QAAQG,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMf,QAAQe,MAAM,sBAAuB,MAAM,MAAMd,QAAQc,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMZ,QAAQe,MAAM,gBAAiB,MAAM,MAAMd,QAAQc,MAAM,YAAYrB,iBAAiBsB,KAAKlB;AACxG;AAEA,iFAAiF;AACjF,eAAewB,MAAMR,MAAoB;IACvCA,OAAOM,MAAM,CAACG,cAAc;IAC5BT,OAAOU,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB,EAAEC,OAA0D;IACrH,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMxB,WAAU;IACxC,EAAE,OAAOyB,KAAK;QACZ,IAAIA,eAAenC,YAAY,MAAMmC;QACrC,MAAM,IAAInC,WAAW,4BAA4B,CAAC,yBAAyB,EAAES,eAAe,UAAU,EAAE,AAAC0B,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIZ;IACJ,IAAII;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMM,eAAeG,MAAM,CAACN,QAAQE,QAAQK,aAAa,GAAG;YAAEC,aAAa;QAAY,IAAI;YAAEC,+BAA+B;QAAS;QAChJhB,SAAS,MAAMI,SAASE,OAAO;IACjC,EAAE,OAAOK,KAAK;QACZ,wFAAwF;QACxFP,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI7B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE+B,OAAO,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMxB,kBAAkBY;QACxB,MAAML,OAAOb,iBAAiBkB;QAC9B,OAAO;YAAEN,QAAQ;gBAAEU;gBAAUJ;YAAO;YAAGL;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMT,MAAM;YAAEE;YAAUJ;QAAO;QAC/B,MAAMW;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMM,oBAA+C;IAC1DC,UAAU3B;IACV4B,eAAe3B;IACf4B,kBAAkBjC;IAClBmB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFmB,UAAU,CAACV,MAAQ,2DAA2DW,IAAI,CAACX,IAAIC,OAAO;IAC9FnB;IACA8B,gBAAgBvC;IAChBwC,oBAAoBzC;IACpBM,aAAa,CAACK,QAAQC,OAASN,YAAYK,OAAOU,QAAQ,EAAEV,OAAOM,MAAM,EAAEL;AAC7E,EAAE;AAEF,OAAO,eAAe8B,WAAW7B,GAAmB,EAAEa,OAA6B;IACjF,OAAO9B,gBAAgBiB,KAAKqB,mBAAmBR;AACjD"}

package/dist/esm/store/duckdb/reconcile.js

@@ -32,8 +32,8 @@ async function reconcileContent(conn, touched, docs, _delta) {
     // The delete above cleared every touched path, and duckdb holds the cache file for its whole
     // connection, so no second writer can have landed one of these rows: nothing can conflict.
     await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));
-    // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
-    // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.
+    // Build prepares the FTS index after reconcile. This persisted mark commits with the content
+    // change, so an interrupted build cannot expose stale lexical results.
     await markContentStale(conn);
 }
 // DuckDB rejects more than one ALTER command per statement ("Parser Error: Only one ALTER

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here. Awaited so the persisted mark (lexical.ts) commits with this same transaction.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","appendRows","quoteIdent","markContentStale","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","addColumns","names","exec","name","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AAEtD,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,MAAMC,qBAAqB,CAAC,qBAAqB,EAAED,gBAAgBE,GAAG,CAACJ,YAAYK,IAAI,CAAC,MAAM,qBAAqB,CAAC;AAEpH,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQZ,GAAG,CAAC,CAACiB,IAAM;YAACA;SAAE;IAE1B,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAMtB,WAAWgB,MAAM,WAAWb,iBAAiBC,oBAAoBc,KAAKb,GAAG,CAACE;IAChF,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAML,iBAAiBc;AACzB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeO,WAAWP,IAAgB,EAAEQ,KAAe;IACzD,IAAIA,MAAMJ,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKS,IAAI,CAACD,MAAMnB,GAAG,CAAC,CAACqB,OAAS,CAAC,mCAAmC,EAAEzB,WAAWyB,MAAM,QAAQ,CAAC,EAAEpB,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMqB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhB,yBAAyB;YACnC,MAAM,IAAIf,WAAW,gBAAgB,CAAC,uBAAuB,EAAE+B,MAAM,8CAA8C,EAAEhB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAS;IACAR;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // Build prepares the FTS index after reconcile. This persisted mark commits with the content\n  // change, so an interrupted build cannot expose stale lexical results.\n  await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","appendRows","quoteIdent","markContentStale","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","addColumns","names","exec","name","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AAEtD,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,MAAMC,qBAAqB,CAAC,qBAAqB,EAAED,gBAAgBE,GAAG,CAACJ,YAAYK,IAAI,CAAC,MAAM,qBAAqB,CAAC;AAEpH,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQZ,GAAG,CAAC,CAACiB,IAAM;YAACA;SAAE;IAE1B,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAMtB,WAAWgB,MAAM,WAAWb,iBAAiBC,oBAAoBc,KAAKb,GAAG,CAACE;IAChF,6FAA6F;IAC7F,uEAAuE;IACvE,MAAML,iBAAiBc;AACzB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeO,WAAWP,IAAgB,EAAEQ,KAAe;IACzD,IAAIA,MAAMJ,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKS,IAAI,CAACD,MAAMnB,GAAG,CAAC,CAACqB,OAAS,CAAC,mCAAmC,EAAEzB,WAAWyB,MAAM,QAAQ,CAAC,EAAEpB,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMqB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhB,yBAAyB;YACnC,MAAM,IAAIf,WAAW,gBAAgB,CAAC,uBAAuB,EAAE+B,MAAM,8CAA8C,EAAEhB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAS;IACAR;AACF,EAAE"}

package/dist/esm/store/index.d.ts

@@ -1,11 +1,30 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { OpenResult } from './open.js';
-export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
-export { clearCache } from './cache.js';
-export type { OpenResult } from './open.js';
+import { clearCache } from './cache.js';
+import type { InternalOpenOptions, OpenOptions, OpenResult } from './open.js';
+import type { Stages } from './stages.js';
+export declare function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult>;
+export declare function openStore(cfg: ResolvedConfig, options?: OpenOptions): Promise<OpenResult>;
+export declare function lexicalReadiness(cfg: ResolvedConfig): Promise<{
+    ready: boolean;
+    error: string | null;
+}>;
+export interface BuildOptions {
+    force?: boolean;
+}
+export interface BuildResult {
+    cfg: ResolvedConfig;
+    dbPath: string;
+    parsed: number;
+    warnings: string[];
+    stages: Stages;
+}
+export declare function buildIndex(cfg: ResolvedConfig, options?: BuildOptions): Promise<BuildResult>;
+export declare function storeFilename(cfg: ResolvedConfig): string;
+export type { OpenOptions, OpenResult } from './open.js';
 export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
 export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Stages } from './stages.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
 export { CAPABILITY_NAMES } from './types.js';
+export { clearCache };

package/dist/esm/store/index.js

@@ -1,26 +1,28 @@
-// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything
-// beyond frontmatter + content lives in src/features/.
+// Rows -> a backing store: registry + the public open/build boundary. Parsing lives in scan.ts;
+// everything beyond frontmatter + content lives in src/features/.
 import { anyPresetEmbeds, storeName } from '../config/index.js';
 import { SenseError } from '../errors.js';
-import { openDuckdb } from './duckdb/open.js';
+import { clearCache } from './cache.js';
+import { DB_FILENAME as DUCKDB_FILENAME, openDuckdb } from './duckdb/open.js';
 import { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.js';
-import { openSqlite } from './sqlite/open.js';
+import { openSqlite, DB_FILENAME as SQLITE_FILENAME } from './sqlite/open.js';
 import { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.js';
-import { openTurso } from './turso/open.js';
+import { openTurso, DB_FILENAME as TURSO_FILENAME } from './turso/open.js';
 import { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.js';
-// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile
-// work for a config that cannot be satisfied), then the store itself opens.
 const REGISTRY = {
     sqlite: {
         capabilities: SQLITE_CAPABILITIES,
+        filename: SQLITE_FILENAME,
         open: openSqlite
     },
     duckdb: {
         capabilities: DUCKDB_CAPABILITIES,
+        filename: DUCKDB_FILENAME,
         open: openDuckdb
     },
     turso: {
         capabilities: TURSO_CAPABILITIES,
+        filename: TURSO_FILENAME,
         open: openTurso
     }
 };
@@ -29,18 +31,81 @@ function entryFor(name) {
     if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store "${name}"; available: ${Object.keys(REGISTRY).join(', ')}`);
     return entry;
 }
-export async function openStore(cfg) {
-    const name = storeName(cfg);
-    const entry = entryFor(name);
-    // The one unambiguous, config-level capability need: an embed block some preset actually uses
-    // for vectors. Lexical search has no such signal at this level (every preset defaults to "words"), so a store lacking it fails loudly at first use.
-    if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
+function requirementsFor(cfg) {
+    const requirements = new Set([
+        'core',
+        'lexical'
+    ]);
+    if (anyPresetEmbeds(cfg)) requirements.add('vectors');
+    return requirements;
+}
+function validateRequirements(cfg, entry, requirements) {
+    if (requirements.has('vectors') && anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
+        const name = storeName(cfg);
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "vectors" in this build, but this config's "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)`);
     }
-    return entry.open(cfg);
 }
-export { clearCache } from './cache.js';
+export async function openStoreFor(cfg, options) {
+    var _options_requirements;
+    const entry = entryFor(storeName(cfg));
+    validateRequirements(cfg, entry, (_options_requirements = options.requirements) !== null && _options_requirements !== void 0 ? _options_requirements : new Set([
+        'core'
+    ]));
+    return entry.open(cfg, options);
+}
+export async function openStore(cfg, options = {}) {
+    const buildEnabled = options.build !== false;
+    return openStoreFor(cfg, {
+        build: buildEnabled,
+        requirements: buildEnabled ? requirementsFor(cfg) : new Set([
+            'core'
+        ])
+    });
+}
+export async function lexicalReadiness(cfg) {
+    let opened;
+    try {
+        opened = await openStoreFor(cfg, {
+            build: false,
+            requirements: new Set([
+                'core',
+                'lexical'
+            ])
+        });
+    } catch (err) {
+        if (err instanceof SenseError && err.code === 'INDEX_NOT_READY') return {
+            ready: false,
+            error: err.message
+        };
+        throw err;
+    }
+    try {
+        return {
+            ready: true,
+            error: null
+        };
+    } finally{
+        await opened.store.close();
+    }
+}
+export async function buildIndex(cfg, options = {}) {
+    if (options.force) clearCache(cfg);
+    const opened = await openStoreFor(cfg, {
+        build: true,
+        requirements: requirementsFor(cfg)
+    });
+    try {
+        const { store: _store, ...result } = opened;
+        return result;
+    } finally{
+        await opened.store.close();
+    }
+}
+export function storeFilename(cfg) {
+    return entryFor(storeName(cfg)).filename;
+}
 export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
 export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export { CAPABILITY_NAMES } from './types.js';
+export { clearCache };

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\nexport { clearCache } from './cache.ts';\nexport type { OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Stages } from './stages.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","clearCache","docCount","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,SAASE,UAAU,QAAQ,aAAa;AAExC,SAASC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,WAAW,EAAEC,cAAc,QAAQ,mBAAmB;AAG/D,SAASC,gBAAgB,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + the public open/build boundary. Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { clearCache } from './cache.ts';\nimport { DB_FILENAME as DUCKDB_FILENAME, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { BuildRequirement, InternalOpenOptions, OpenOptions, OpenResult } from './open.ts';\nimport { openSqlite, DB_FILENAME as SQLITE_FILENAME } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Stages } from './stages.ts';\nimport { openTurso, DB_FILENAME as TURSO_FILENAME } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  filename: string;\n  open: (cfg: ResolvedConfig, options?: InternalOpenOptions) => Promise<OpenResult>;\n}\n\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, filename: SQLITE_FILENAME, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, filename: DUCKDB_FILENAME, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, filename: TURSO_FILENAME, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nfunction requirementsFor(cfg: ResolvedConfig): Set<BuildRequirement> {\n  const requirements = new Set<BuildRequirement>(['core', 'lexical']);\n  if (anyPresetEmbeds(cfg)) requirements.add('vectors');\n  return requirements;\n}\n\nfunction validateRequirements(cfg: ResolvedConfig, entry: StoreEntry, requirements: ReadonlySet<BuildRequirement>): void {\n  if (requirements.has('vectors') && anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    const name = storeName(cfg);\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n}\n\nexport async function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult> {\n  const entry = entryFor(storeName(cfg));\n  validateRequirements(cfg, entry, options.requirements ?? new Set<BuildRequirement>(['core']));\n  return entry.open(cfg, options);\n}\n\nexport async function openStore(cfg: ResolvedConfig, options: OpenOptions = {}): Promise<OpenResult> {\n  const buildEnabled = options.build !== false;\n  return openStoreFor(cfg, {\n    build: buildEnabled,\n    requirements: buildEnabled ? requirementsFor(cfg) : new Set<BuildRequirement>(['core']),\n  });\n}\n\nexport async function lexicalReadiness(cfg: ResolvedConfig): Promise<{ ready: boolean; error: string | null }> {\n  let opened: OpenResult;\n  try {\n    opened = await openStoreFor(cfg, { build: false, requirements: new Set<BuildRequirement>(['core', 'lexical']) });\n  } catch (err) {\n    if (err instanceof SenseError && err.code === 'INDEX_NOT_READY') return { ready: false, error: err.message };\n    throw err;\n  }\n  try {\n    return { ready: true, error: null };\n  } finally {\n    await opened.store.close();\n  }\n}\n\nexport interface BuildOptions {\n  force?: boolean;\n}\n\nexport interface BuildResult {\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\nexport async function buildIndex(cfg: ResolvedConfig, options: BuildOptions = {}): Promise<BuildResult> {\n  if (options.force) clearCache(cfg);\n  const opened = await openStoreFor(cfg, { build: true, requirements: requirementsFor(cfg) });\n  try {\n    const { store: _store, ...result } = opened;\n    return result;\n  } finally {\n    await opened.store.close();\n  }\n}\n\nexport function storeFilename(cfg: ResolvedConfig): string {\n  return entryFor(storeName(cfg)).filename;\n}\n\nexport type { OpenOptions, OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Stages } from './stages.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\nexport { clearCache };\n"],"names":["anyPresetEmbeds","storeName","SenseError","clearCache","DB_FILENAME","DUCKDB_FILENAME","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_FILENAME","SQLITE_CAPABILITIES","openTurso","TURSO_FILENAME","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","filename","open","duckdb","turso","entryFor","name","entry","Object","keys","join","requirementsFor","cfg","requirements","Set","add","validateRequirements","has","openStoreFor","options","openStore","buildEnabled","build","lexicalReadiness","opened","err","code","ready","error","message","store","close","buildIndex","force","_store","result","storeFilename","docCount","getMeta","setMeta","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,gGAAgG;AAChG,kEAAkE;AAGlE,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,aAAa;AACxC,SAASC,eAAeC,eAAe,EAAEC,UAAU,QAAQ,mBAAmB;AAC9E,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,EAAEL,eAAeM,eAAe,QAAQ,mBAAmB;AAC9E,SAASH,gBAAgBI,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,SAAS,EAAER,eAAeS,cAAc,QAAQ,kBAAkB;AAC3E,SAASN,gBAAgBO,kBAAkB,QAAQ,mBAAmB;AAStE,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcN;QAAqBO,UAAUR;QAAiBS,MAAMV;IAAW;IACzFW,QAAQ;QAAEH,cAAcT;QAAqBU,UAAUb;QAAiBc,MAAMb;IAAW;IACzFe,OAAO;QAAEJ,cAAcH;QAAoBI,UAAUL;QAAgBM,MAAMP;IAAU;AACvF;AAEA,SAASU,SAASC,IAAe;IAC/B,MAAMC,QAAQT,QAAQ,CAACQ,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAItB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEqB,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACX,UAAUY,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,SAASI,gBAAgBC,GAAmB;IAC1C,MAAMC,eAAe,IAAIC,IAAsB;QAAC;QAAQ;KAAU;IAClE,IAAI/B,gBAAgB6B,MAAMC,aAAaE,GAAG,CAAC;IAC3C,OAAOF;AACT;AAEA,SAASG,qBAAqBJ,GAAmB,EAAEL,KAAiB,EAAEM,YAA2C;IAC/G,IAAIA,aAAaI,GAAG,CAAC,cAAclC,gBAAgB6B,QAAQ,CAACL,MAAMP,YAAY,CAACiB,GAAG,CAAC,YAAY;QAC7F,MAAMX,OAAOtB,UAAU4B;QACvB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;AACF;AAEA,OAAO,eAAeY,aAAaN,GAAmB,EAAEO,OAA4B;QAEjDA;IADjC,MAAMZ,QAAQF,SAASrB,UAAU4B;IACjCI,qBAAqBJ,KAAKL,QAAOY,wBAAAA,QAAQN,YAAY,cAApBM,mCAAAA,wBAAwB,IAAIL,IAAsB;QAAC;KAAO;IAC3F,OAAOP,MAAML,IAAI,CAACU,KAAKO;AACzB;AAEA,OAAO,eAAeC,UAAUR,GAAmB,EAAEO,UAAuB,CAAC,CAAC;IAC5E,MAAME,eAAeF,QAAQG,KAAK,KAAK;IACvC,OAAOJ,aAAaN,KAAK;QACvBU,OAAOD;QACPR,cAAcQ,eAAeV,gBAAgBC,OAAO,IAAIE,IAAsB;YAAC;SAAO;IACxF;AACF;AAEA,OAAO,eAAeS,iBAAiBX,GAAmB;IACxD,IAAIY;IACJ,IAAI;QACFA,SAAS,MAAMN,aAAaN,KAAK;YAAEU,OAAO;YAAOT,cAAc,IAAIC,IAAsB;gBAAC;gBAAQ;aAAU;QAAE;IAChH,EAAE,OAAOW,KAAK;QACZ,IAAIA,eAAexC,cAAcwC,IAAIC,IAAI,KAAK,mBAAmB,OAAO;YAAEC,OAAO;YAAOC,OAAOH,IAAII,OAAO;QAAC;QAC3G,MAAMJ;IACR;IACA,IAAI;QACF,OAAO;YAAEE,OAAO;YAAMC,OAAO;QAAK;IACpC,SAAU;QACR,MAAMJ,OAAOM,KAAK,CAACC,KAAK;IAC1B;AACF;AAcA,OAAO,eAAeC,WAAWpB,GAAmB,EAAEO,UAAwB,CAAC,CAAC;IAC9E,IAAIA,QAAQc,KAAK,EAAE/C,WAAW0B;IAC9B,MAAMY,SAAS,MAAMN,aAAaN,KAAK;QAAEU,OAAO;QAAMT,cAAcF,gBAAgBC;IAAK;IACzF,IAAI;QACF,MAAM,EAAEkB,OAAOI,MAAM,EAAE,GAAGC,QAAQ,GAAGX;QACrC,OAAOW;IACT,SAAU;QACR,MAAMX,OAAOM,KAAK,CAACC,KAAK;IAC1B;AACF;AAEA,OAAO,SAASK,cAAcxB,GAAmB;IAC/C,OAAOP,SAASrB,UAAU4B,MAAMX,QAAQ;AAC1C;AAGA,SAASoC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASpD,WAAW,EAAEqD,cAAc,QAAQ,mBAAmB;AAG/D,SAASC,gBAAgB,QAAQ,aAAa;AAC9C,SAASvD,UAAU,GAAG"}

package/dist/esm/store/open.d.ts

@@ -1,6 +1,13 @@
 import type { ResolvedConfig } from '../config/index.js';
 import type { Stages } from './stages.js';
 import type { OpenDialect, Store } from './types.js';
+export type BuildRequirement = 'core' | 'lexical' | 'vectors';
+export interface OpenOptions {
+    build?: boolean;
+}
+export interface InternalOpenOptions extends OpenOptions {
+    requirements?: ReadonlySet<BuildRequirement>;
+}
 export interface OpenResult {
     store: Store;
     cfg: ResolvedConfig;
@@ -9,5 +16,6 @@ export interface OpenResult {
     warnings: string[];
     stages: Stages;
 }
-export declare function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult>;
+export declare function prepareDocumentEmbeddings(store: Store, cfg: ResolvedConfig, allowed?: ReadonlySet<string>): Promise<void>;
+export declare function openWithDialect<Handle>(input: ResolvedConfig, dialect: OpenDialect<Handle>, options?: InternalOpenOptions): Promise<OpenResult>;
 export declare function docCount(store: Store): Promise<number>;