sensemaking 0.19.2 → 0.20.0

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, duckdbDialect);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openDuckdb","ensureSchema","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","DuckDBInstance","err","duckdb","instance","closed","version","features","wantFeatures","reason","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","closeSync","registerFunctions","createConnection","console","error","disconnectSync","clearCache","reconcile","duckdbDialect","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAmBaA;eAAAA;;QAGAC;eAAAA;;QA+FSC;eAAAA;;;sBArHI;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;2BACD;wBACO;4BAEA;wBACS;4BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAU9B,8LAA8L;AAC9L,2JAA2J;AAC3J,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAKlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMW,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,QAAQX,GAAmB;;YAClCY,UAEAC,QAIFC,gBAGKC,KAKLC,QACAC,UAMKF,MAOLG,QAGInB,MAIAoB,SACAC,UACAC,cAIEC,QAYqB,MAArBC,QAAQC,UAETT;;;;oBAvDHH,WAAWa,IAAAA,cAAI,EAACzB,IAAI0B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUjB;;;;;;;;;oBAMN;;wBAAMmC,IAAAA,mBAAS;;;oBAAlChB,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAMnB,eAAeoB,MAAM,CAACrB,QAAQ;4BAAEsB,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KlB,WAAW;oBACF;;wBAAMA,SAASN,OAAO;;;oBAA/BK,SAAS;;;;;;oBACFD;oBACP,wFAAwF;oBACxFE,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACnB,MAAM,IAAIL,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAErH,2FAA2F;oBAC3F,uFAAuF;oBACnFf,SAAS;;;;;;;;;oBAEX;;wBAAMmB,IAAAA,iCAAiB,EAACrB;;;oBAAxB;oBACMjB,OAAOuC,IAAAA,8BAAgB,EAACtB;oBAE9B;;wBAAMjB,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACR,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMZ,IAAAA,iBAAO,EAACR,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeZ,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;oBACnD,IAAI,AAACS,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;wBACxG,4FAA4F;wBAC5F,wFAAwF;wBAClFC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;wBACnH2C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPlB,QAAO;wBAC/B,gFAAgF;wBAChFJ,SAAS;wBACTF,OAAOyB,cAAc;wBACrBxB,SAASmB,SAAS;wBAClBM,IAAAA,mBAAU,EAAC1C;wBACX;;4BAAOW,QAAQX;;oBACjB;oBAEA;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAE6B;;wBAAM2C,IAAAA,sBAAS,EAAC5C,MAAMC,KAAKA,IAAI0B,OAAO,EAAEkB,2BAAa;;;oBAArD,OAAA,eAArBrB,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAChB;;wBAAO;4BAAEP,UAAAA;4BAAUD,QAAAA;4BAAQjB,MAAAA;4BAAMC,KAAAA;4BAAKa,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBACxDT;oBACP,IAAI,CAACG,QAAQ;wBACXF,OAAOyB,cAAc;wBACrBxB,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACrB;oBACA,MAAMrB;;;;;;;IAEV;;AAEO,SAAelB,WAAWG,GAAmB;;YAC6B,MAAvEiB,UAAUD,QAAQjB,MAAW8C,aAAahC,QAAQU,QAAQC,UAC5DsB;;;;oBADyE;;wBAAMnC,QAAQX;;;oBAAd,OAAA,eAAvEiB,WAAuE,KAAvEA,UAAUD,SAA6D,KAA7DA,QAAQjB,OAAqD,KAArDA,MAAW8C,cAA0C,KAA/C7C,KAAkBa,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC5DsB,QAAQC,IAAAA,oBAAW,EAAC9B,UAAUD,QAAQjB,MAAM8C,aAAaA,YAAYnB,OAAO;oBAClF,0FAA0F;oBAC1FsB,IAAAA,yBAAc,EAACjD,MAAM+C;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAO9C,KAAK6C;4BAAahC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAUSC;eAAAA;;;uBAnGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,2JAA2J;AAC3J,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACAd,cAAAA;IACAqC,aAAa,SAAbA,YAActB,QAAQb;eAASmC,IAAAA,oBAAW,EAACtB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOmC,IAAAA,uBAAe,EAACnC,KAAKL;;;IAC9B"}

package/dist/cjs/store/duckdb/reconcile.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "duckdbDialect", {
     }
 });
 var _errorsts = require("../../errors.js");
+var _lexicalts = require("./lexical.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -185,6 +186,9 @@ function reconcileContent(conn, touched, docs, _delta) {
                     _state.sent();
                     _state.label = 4;
                 case 4:
+                    // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
+                    // not here (lexical.ts's FtsIndexState).
+                    (0, _lexicalts.markContentStale)(conn);
                     return [
                         2
                     ];

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n  // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n  columnTypeSuffix: () => ' VARIANT',\n  reconcileContent,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","beginMode","checkColumnLimit","count","SenseError","columnTypeSuffix"],"mappings":";;;;+BA+BaA;;;eAAAA;;;wBA/Bc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK3B,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;;;;;;IACvB;;AAEO,IAAMF,gBAAkC;IAC7CmB,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQZ,yBAAyB;YACnC,MAAM,IAAIa,oBAAU,CAAC,gBAAgB,AAAC,0BAA+Eb,OAAtDY,OAAM,kDAAwE,OAAxBZ,yBAAwB;QAC/I;IACF;IACA,yFAAyF;IACzF,2FAA2F;IAC3Fc,kBAAkB,SAAlBA;eAAwB;;IACxBb,kBAAAA;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n  // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n  columnTypeSuffix: () => ' VARIANT',\n  reconcileContent,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","markContentStale","beginMode","checkColumnLimit","count","SenseError","columnTypeSuffix"],"mappings":";;;;+BAmCaA;;;eAAAA;;;wBAnCc;yBAIM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;oBACrB,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEO,IAAMX,gBAAkC;IAC7CoB,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQb,yBAAyB;YACnC,MAAM,IAAIc,oBAAU,CAAC,gBAAgB,AAAC,0BAA+Ed,OAAtDa,OAAM,kDAAwE,OAAxBb,yBAAwB;QAC/I;IACF;IACA,yFAAyF;IACzF,2FAA2F;IAC3Fe,kBAAkB,SAAlBA;eAAwB;;IACxBd,kBAAAA;AACF"}

package/dist/cjs/store/duckdb/store.d.cts

@@ -1,5 +1,4 @@
 import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store;

package/dist/cjs/store/duckdb/store.d.ts

@@ -1,5 +1,4 @@
 import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store;

package/dist/cjs/store/duckdb/store.js

@@ -17,14 +17,12 @@ _export(exports, {
     }
 });
 var _typests = require("../../embed/types.js");
-var _reconcilets = require("../reconcile.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _connectionts = require("./connection.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
-var _reconcilets1 = require("./reconcile.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -270,7 +268,7 @@ var CAPABILITIES = new Set([
     'vectors',
     'sql-functions'
 ]);
-function createStore(instance, duckdb, conn, cfg, baseDir) {
+function createStore(instance, duckdb, conn) {
     var lex = (0, _lexicalts.createLexicalIndex)(conn);
     return {
         name: 'duckdb',
@@ -331,29 +329,6 @@ function createStore(instance, duckdb, conn, cfg, baseDir) {
                 });
             })();
         },
-        reconcile: function reconcile() {
-            return _async_to_generator(function() {
-                var result;
-                return _ts_generator(this, function(_state) {
-                    switch(_state.label){
-                        case 0:
-                            return [
-                                4,
-                                (0, _reconcilets.reconcile)(conn, cfg, baseDir, _reconcilets1.duckdbDialect)
-                            ];
-                        case 1:
-                            result = _state.sent();
-                            // content may have changed; the fts index is rebuilt lazily, on the next lexical query
-                            // that needs it, not here (see lexical.ts's FtsIndexState).
-                            lex.markStale();
-                            return [
-                                2,
-                                result
-                            ];
-                    }
-                });
-            })();
-        },
         docs: {
             columns: function columns() {
                 return _async_to_generator(function() {

package/dist/cjs/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir, duckdbDialect);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","cfg","baseDir","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","result","duckdbDialect","markStale","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAIGC;eAAAA;;;uBAlBW;2BACD;wBACC;6BACK;yBAEU;4BACb;4BACF;yBACQ;4BACL;0BACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB;AAIxG,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,IAAMC,MAAMC,IAAAA,6BAAkB,EAACJ;IAC/B,OAAO;QACLK,MAAM;QACNC,cAAcX;QACRY,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMR,KAAKO,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOR,KAAKS,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMX,KAAKU,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACd,MAAMa;;;YAC/B;;QACME,WAAN,SAAMA;;oBACEC;;;;4BAAS;;gCAAMD,IAAAA,sBAAS,EAACf,MAAMC,KAAKC,SAASe,2BAAa;;;4BAA1DD,SAAS;4BACf,uFAAuF;4BACvF,4DAA4D;4BAC5Db,IAAIe,SAAS;4BACb;;gCAAOF;;;;YACT;;QACAG,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACrB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAsB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACtB,MAAMoB,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOtB,IAAIsB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAAC5B;;YAC3B6B,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAChC,QAAQC,MAAMgC,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACvC,QAAQmC,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAAC3C,QAAQC,MAAMgC,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAAC5C,MAAMwC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGrC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBuC;;;;gCAAW;;oCAAMhD,OAAOU,OAAO,CAACD;;;gCAAhCuC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE9C,MAAM0C,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjFlE,OAAOmE,cAAc;oBACrBpE,SAASqE,SAAS;;;;;YACpB;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACb;4BACF;yBACQ;0BAC2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB;AAIxG,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,IAAMC,MAAMC,IAAAA,6BAAkB,EAACF;IAC/B,OAAO;QACLG,MAAM;QACNC,cAAcT;QACRU,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACf;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAgB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAAChB,MAAMc,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOlB,IAAIkB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACtB;;YAC3BuB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,QAAQC,MAAM0B,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,QAAQ6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAACrC,QAAQC,MAAM0B,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAACtC,MAAMkC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGjC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBmC;;;;gCAAW;;oCAAM1C,OAAOQ,OAAO,CAACD;;;gCAAhCmC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE1C,MAAMsC,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjF5D,OAAO6D,cAAc;oBACrB9D,SAAS+D,SAAS;;;;;YACpB;;IACF;AACF"}

package/dist/cjs/store/index.d.cts

@@ -1,10 +1,15 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { OpenResult } from './sqlite/open.js';
+import type { Builder } from './builder.js';
+import type { OpenResult } from './open.js';
 export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & {
+    builder: Builder;
+}>;
 export declare function requireWatchConcurrency(cfg: ResolvedConfig): void;
 export { clearCache } from './cache.js';
+export type { OpenResult } from './open.js';
+export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
-export type { OpenResult } from './sqlite/open.js';
-export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
 export { CAPABILITY_NAMES } from './types.js';

package/dist/cjs/store/index.d.ts

@@ -1,10 +1,15 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { OpenResult } from './sqlite/open.js';
+import type { Builder } from './builder.js';
+import type { OpenResult } from './open.js';
 export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & {
+    builder: Builder;
+}>;
 export declare function requireWatchConcurrency(cfg: ResolvedConfig): void;
 export { clearCache } from './cache.js';
+export type { OpenResult } from './open.js';
+export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
-export type { OpenResult } from './sqlite/open.js';
-export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
 export { CAPABILITY_NAMES } from './types.js';

package/dist/cjs/store/index.js

@@ -15,10 +15,10 @@ _export(exports, {
         return _typests.CAPABILITY_NAMES;
     },
     get DB_FILENAME () {
-        return _opents1.DB_FILENAME;
+        return _opents2.DB_FILENAME;
     },
     get SCHEMA_VERSION () {
-        return _opents1.SCHEMA_VERSION;
+        return _opents2.SCHEMA_VERSION;
     },
     get clearCache () {
         return _cachets.clearCache;
@@ -32,6 +32,9 @@ _export(exports, {
     get openStore () {
         return openStore;
     },
+    get openStoreForWatch () {
+        return openStoreForWatch;
+    },
     get requireWatchConcurrency () {
         return requireWatchConcurrency;
     },
@@ -43,9 +46,10 @@ var _indexts = require("../config/index.js");
 var _errorsts = require("../errors.js");
 var _opents = require("./duckdb/open.js");
 var _storets = require("./duckdb/store.js");
-var _opents1 = require("./sqlite/open.js");
+var _opents1 = require("./open.js");
+var _opents2 = require("./sqlite/open.js");
 var _storets1 = require("./sqlite/store.js");
-var _opents2 = require("./turso/open.js");
+var _opents3 = require("./turso/open.js");
 var _storets2 = require("./turso/store.js");
 var _cachets = require("./cache.js");
 var _sharedts = require("./shared.js");
@@ -180,15 +184,24 @@ function _ts_generator(thisArg, body) {
 var REGISTRY = {
     sqlite: {
         capabilities: _storets1.CAPABILITIES,
-        open: _opents1.openSqlite
+        open: _opents2.openSqlite,
+        openForWatch: function openForWatch(cfg) {
+            return (0, _opents1.openWithBuilder)(cfg, _opents2.sqliteOpenDialect);
+        }
     },
     duckdb: {
         capabilities: _storets.CAPABILITIES,
-        open: _opents.openDuckdb
+        open: _opents.openDuckdb,
+        openForWatch: function openForWatch(cfg) {
+            return (0, _opents1.openWithBuilder)(cfg, _opents.duckdbOpenDialect);
+        }
     },
     turso: {
         capabilities: _storets2.CAPABILITIES,
-        open: _opents2.openTurso
+        open: _opents3.openTurso,
+        openForWatch: function openForWatch(cfg) {
+            return (0, _opents1.openWithBuilder)(cfg, _opents3.tursoOpenDialect);
+        }
     }
 };
 function entryFor(name) {
@@ -214,6 +227,22 @@ function openStore(cfg) {
         });
     })();
 }
+function openStoreForWatch(cfg) {
+    return _async_to_generator(function() {
+        var name, entry;
+        return _ts_generator(this, function(_state) {
+            name = (0, _indexts.storeName)(cfg);
+            entry = entryFor(name);
+            if ((0, _indexts.anyPresetEmbeds)(cfg) && !entry.capabilities.has('vectors')) {
+                throw new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "'.concat(name, '" does not implement "vectors" in this build, but this config\'s "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)'));
+            }
+            return [
+                2,
+                entry.openForWatch(cfg)
+            ];
+        });
+    })();
+}
 function requireWatchConcurrency(cfg) {
     var name = (0, _indexts.storeName)(cfg);
     if (!entryFor(name).capabilities.has('watch-concurrency')) {

package/dist/cjs/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["CAPABILITY_NAMES","DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","REGISTRY","sqlite","capabilities","SQLITE_CAPABILITIES","open","openSqlite","duckdb","DUCKDB_CAPABILITIES","openDuckdb","turso","TURSO_CAPABILITIES","openTurso","entryFor","name","entry","SenseError","Object","keys","join","cfg","storeName","anyPresetEmbeds","has"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;;;;;;;;;;;;QA4D9CA;eAAAA,yBAAgB;;QAFhBC;eAAAA,oBAAW;;QAAYC;eAAAA,uBAAc;;QAHrCC;eAAAA,mBAAU;;QAGGC;eAAAA,iBAAQ;;QAFrBC;eAAAA,iBAAO;;QAvBMC;eAAAA;;QAeNC;eAAAA;;QAQEC;eAAAA,iBAAO;;;uBArDkB;wBAChB;sBACA;uBACyB;uBAEzB;wBACyB;uBAC1B;wBACyB;uBA4CxB;wBACM;uBAIA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAzCjC,kGAAkG;AAClG,4EAA4E;AAC5E,IAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcC,sBAAmB;QAAEC,MAAMC,mBAAU;IAAC;IAC9DC,QAAQ;QAAEJ,cAAcK,qBAAmB;QAAEH,MAAMI,kBAAU;IAAC;IAC9DC,OAAO;QAAEP,cAAcQ,sBAAkB;QAAEN,MAAMO,kBAAS;IAAC;AAC7D;AAEA,SAASC,SAASC,IAAe;IAC/B,IAAMC,QAAQd,QAAQ,CAACa,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIC,oBAAU,CAAC,iBAAiB,AAAC,0BAA8CC,OAArBH,MAAK,kBAAiD,OAAjCG,OAAOC,IAAI,CAACjB,UAAUkB,IAAI,CAAC;IAC5H,OAAOJ;AACT;AAEO,SAAejB,UAAUsB,GAAmB;;YAC3CN,MACAC;;YADAD,OAAOO,IAAAA,kBAAS,EAACD;YACjBL,QAAQF,SAASC;YAEvB,8FAA8F;YAC9F,oJAAoJ;YACpJ,IAAIQ,IAAAA,wBAAe,EAACF,QAAQ,CAACL,MAAMZ,YAAY,CAACoB,GAAG,CAAC,YAAY;gBAC9D,MAAM,IAAIP,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;YAClE;YAEA;;gBAAOC,MAAMV,IAAI,CAACe;;;IACpB;;AAIO,SAASrB,wBAAwBqB,GAAmB;IACzD,IAAMN,OAAOO,IAAAA,kBAAS,EAACD;IACvB,IAAI,CAACP,SAASC,MAAMX,YAAY,CAACoB,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAIP,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;IAClE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Builder } from './builder.ts';\nimport { duckdbOpenDialect, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './open.ts';\nimport { openWithBuilder } from './open.ts';\nimport { openSqlite, sqliteOpenDialect } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso, tursoOpenDialect } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n  // Same open, but keeps the builder's parse pool alive and hands it back: only a watcher, which\n  // reconciles repeatedly on the same connection, needs this.\n  openForWatch: (cfg: ResolvedConfig) => Promise<OpenResult & { builder: Builder }>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite, openForWatch: (cfg) => openWithBuilder(cfg, sqliteOpenDialect) },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb, openForWatch: (cfg) => openWithBuilder(cfg, duckdbOpenDialect) },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso, openForWatch: (cfg) => openWithBuilder(cfg, tursoOpenDialect) },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// Same capability-checked open as openStore, but for a caller (only src/watch.ts) that must\n// reconcile again later: keeps the builder's pool warm and hands it back instead of closing it.\nexport async function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & { builder: Builder }> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.openForWatch(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport type { OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["CAPABILITY_NAMES","DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","getMeta","openStore","openStoreForWatch","requireWatchConcurrency","setMeta","REGISTRY","sqlite","capabilities","SQLITE_CAPABILITIES","open","openSqlite","openForWatch","cfg","openWithBuilder","sqliteOpenDialect","duckdb","DUCKDB_CAPABILITIES","openDuckdb","duckdbOpenDialect","turso","TURSO_CAPABILITIES","openTurso","tursoOpenDialect","entryFor","name","entry","SenseError","Object","keys","join","storeName","anyPresetEmbeds","has"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;;;;;;;;;;;;QA+E9CA;eAAAA,yBAAgB;;QAFhBC;eAAAA,oBAAW;;QAAEC;eAAAA,uBAAc;;QAJ3BC;eAAAA,mBAAU;;QAEVC;eAAAA,iBAAQ;;QACRC;eAAAA,iBAAO;;QAtCMC;eAAAA;;QAeAC;eAAAA;;QAaNC;eAAAA;;QAUEC;eAAAA,iBAAO;;;uBAzEkB;wBAChB;sBAEmB;uBACM;uBAEpB;uBACc;wBACM;uBACR;wBACO;uBA4DxB;wBAGM;uBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAvDjC,kGAAkG;AAClG,4EAA4E;AAC5E,IAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcC,sBAAmB;QAAEC,MAAMC,mBAAU;QAAEC,cAAc,SAAdA,aAAeC;mBAAQC,IAAAA,wBAAe,EAACD,KAAKE,0BAAiB;;IAAE;IAC9HC,QAAQ;QAAER,cAAcS,qBAAmB;QAAEP,MAAMQ,kBAAU;QAAEN,cAAc,SAAdA,aAAeC;mBAAQC,IAAAA,wBAAe,EAACD,KAAKM,yBAAiB;;IAAE;IAC9HC,OAAO;QAAEZ,cAAca,sBAAkB;QAAEX,MAAMY,kBAAS;QAAEV,cAAc,SAAdA,aAAeC;mBAAQC,IAAAA,wBAAe,EAACD,KAAKU,yBAAgB;;IAAE;AAC5H;AAEA,SAASC,SAASC,IAAe;IAC/B,IAAMC,QAAQpB,QAAQ,CAACmB,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIC,oBAAU,CAAC,iBAAiB,AAAC,0BAA8CC,OAArBH,MAAK,kBAAiD,OAAjCG,OAAOC,IAAI,CAACvB,UAAUwB,IAAI,CAAC;IAC5H,OAAOJ;AACT;AAEO,SAAexB,UAAUW,GAAmB;;YAC3CY,MACAC;;YADAD,OAAOM,IAAAA,kBAAS,EAAClB;YACjBa,QAAQF,SAASC;YAEvB,8FAA8F;YAC9F,oJAAoJ;YACpJ,IAAIO,IAAAA,wBAAe,EAACnB,QAAQ,CAACa,MAAMlB,YAAY,CAACyB,GAAG,CAAC,YAAY;gBAC9D,MAAM,IAAIN,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;YAClE;YAEA;;gBAAOC,MAAMhB,IAAI,CAACG;;;IACpB;;AAIO,SAAeV,kBAAkBU,GAAmB;;YACnDY,MACAC;;YADAD,OAAOM,IAAAA,kBAAS,EAAClB;YACjBa,QAAQF,SAASC;YAEvB,IAAIO,IAAAA,wBAAe,EAACnB,QAAQ,CAACa,MAAMlB,YAAY,CAACyB,GAAG,CAAC,YAAY;gBAC9D,MAAM,IAAIN,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;YAClE;YAEA;;gBAAOC,MAAMd,YAAY,CAACC;;;IAC5B;;AAIO,SAAST,wBAAwBS,GAAmB;IACzD,IAAMY,OAAOM,IAAAA,kBAAS,EAAClB;IACvB,IAAI,CAACW,SAASC,MAAMjB,YAAY,CAACyB,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAIN,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;IAClE;AACF"}

package/dist/cjs/store/open.d.cts

@@ -0,0 +1,15 @@
+import type { ResolvedConfig } from '../config/index.js';
+import type { Builder } from './builder.js';
+import type { OpenDialect, Store } from './types.js';
+export interface OpenResult {
+    store: Store;
+    cfg: ResolvedConfig;
+    dbPath: string;
+    parsed: number;
+    warnings: string[];
+}
+export declare function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult>;
+export declare function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & {
+    builder: Builder;
+}>;
+export declare function docCount(store: Store): Promise<number>;

package/dist/cjs/store/open.d.ts

@@ -0,0 +1,15 @@
+import type { ResolvedConfig } from '../config/index.js';
+import type { Builder } from './builder.js';
+import type { OpenDialect, Store } from './types.js';
+export interface OpenResult {
+    store: Store;
+    cfg: ResolvedConfig;
+    dbPath: string;
+    parsed: number;
+    warnings: string[];
+}
+export declare function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult>;
+export declare function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & {
+    builder: Builder;
+}>;
+export declare function docCount(store: Store): Promise<number>;