sensemaking 0.18.4 → 0.19.1

package/dist/esm/store/duckdb/open.js

@@ -1,36 +1,31 @@
 import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { featureSignature, STATE_DIR } from '../../config/index.js';
+import { rekeyChunkText } from '../../embed/handoff.js';
 import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
+import { clearCache } from '../cache.js';
 import { getMeta, setMeta } from '../shared.js';
-import { clearCache } from '../sqlite/open.js';
 import { createConnection } from './connection.js';
 import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
 import { reconcile } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.duckdb';
-// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately
-// (VARIANT frontmatter columns instead of untyped ones), and the store name already joins the
-// feature signature (see featureSignature), so a config's `store` key switching cleanly
-// rebuilds rather than reusing the other engine's stale cache shape.
+// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
+// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
 export const SCHEMA_VERSION = '2';
-// `content` is a plain table (not FTS-virtual): D1 builds DuckDB's fts index over it lazily,
-// only when a lexical query actually runs (see duckdb/lexical.ts), and reads it directly for
-// contains() substring/phrase/unspaced-script verification either way. No tokenizer resolution
-// -- this store always uses the fts extension's default (porter) stemmer, unlike sqlite's
-// configurable `content.tokenize` (not read here; D1 does not extend that knob to DuckDB).
+// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
+// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.
 async function ensureSchema(conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
     for (const feature of activeFeatures(cfg)){
-        // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL
-        // (see duckdb/vectors.ts). `scale` is kept, unused, so the feature's shared reconcile-time
-        // INSERT/DELETE (src/features/embed.ts) still names a column that exists on both stores.
+        // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,
+        // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.
         if (feature.name === 'embed') {
             await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY ("path", chunk))`);
             continue;
@@ -46,11 +41,8 @@ async function connect(cfg) {
         recursive: true
     });
     const dbPath = join(stateDir, DB_FILENAME);
-    // Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this
-    // optional peer dependency, so nothing in this store's module graph imports it as a value
-    // until a duckdb tree is actually opened (types-only imports elsewhere are erased and cost
-    // nothing either way). Installed on first use if missing, and shared with sql-functions.ts
-    // and vectors.ts (see native.ts's duckdbApi).
+    // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports
+    // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.
     let DuckDBInstance;
     try {
         ({ DuckDBInstance } = await duckdbApi());
@@ -61,10 +53,8 @@ async function connect(cfg) {
     let duckdb;
     let instance;
     try {
-        // On-disk files default to an older storage format for cross-version compatibility, which
-        // rejects VARIANT columns ("VARIANT columns are not supported in storage versions prior to
-        // v1.5.0"); this store's dynamic frontmatter columns need VARIANT (see reconcile.ts), so
-        // the format floor is pinned explicitly. :memory: databases are unaffected either way.
+        // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns ("VARIANT
+        // columns are not supported in storage versions prior to v1.5.0"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.
         instance = await DuckDBInstance.create(dbPath, {
             storage_compatibility_version: 'v1.5.0'
         });
@@ -117,8 +107,11 @@ async function connect(cfg) {
 }
 export async function openDuckdb(cfg) {
     const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
+    const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);
+    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+    rekeyChunkText(conn, store);
     return {
-        store: createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir),
+        store: store,
         cfg: resolvedCfg,
         dbPath,
         parsed,

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { clearCache } from '../sqlite/open.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately\n// (VARIANT frontmatter columns instead of untyped ones), and the store name already joins the\n// feature signature (see featureSignature), so a config's `store` key switching cleanly\n// rebuilds rather than reusing the other engine's stale cache shape.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): D1 builds DuckDB's fts index over it lazily,\n// only when a lexical query actually runs (see duckdb/lexical.ts), and reads it directly for\n// contains() substring/phrase/unspaced-script verification either way. No tokenizer resolution\n// -- this store always uses the fts extension's default (porter) stemmer, unlike sqlite's\n// configurable `content.tokenize` (not read here; D1 does not extend that knob to DuckDB).\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL\n    // (see duckdb/vectors.ts). `scale` is kept, unused, so the feature's shared reconcile-time\n    // INSERT/DELETE (src/features/embed.ts) still names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this\n  // optional peer dependency, so nothing in this store's module graph imports it as a value\n  // until a duckdb tree is actually opened (types-only imports elsewhere are erased and cost\n  // nothing either way). Installed on first use if missing, and shared with sql-functions.ts\n  // and vectors.ts (see native.ts's duckdbApi).\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which\n    // rejects VARIANT columns (\"VARIANT columns are not supported in storage versions prior to\n    // v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (see reconcile.ts), so\n    // the format floor is pinned explicitly. :memory: databases are unaffected either way.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","STORE_DIMS","SenseError","activeFeatures","FEATURES","getMeta","setMeta","clearCache","createConnection","DUCKDB_PACKAGE","duckdbApi","reconcile","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,2FAA2F;AAC3F,8FAA8F;AAC9F,wFAAwF;AACxF,qEAAqE;AACrE,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,6FAA6F;AAC7F,6FAA6F;AAC7F,+FAA+F;AAC/F,0FAA0F;AAC1F,2FAA2F;AAC3F,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWjB,eAAee,KAAM;QACzC,wFAAwF;QACxF,2FAA2F;QAC3F,yFAAyF;QACzF,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAElB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMmB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMV,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKd;AACxG;AAEA,eAAemB,QAAQL,GAAmB;IACxC,MAAMM,WAAW1B,KAAKoB,IAAIO,OAAO,EAAEzB;IACnCH,UAAU2B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS7B,KAAK0B,UAAUV;IAE9B,wFAAwF;IACxF,0FAA0F;IAC1F,2FAA2F;IAC3F,2FAA2F;IAC3F,8CAA8C;IAC9C,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe3B,YAAY,MAAM2B;QACrC,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,0FAA0F;QAC1F,2FAA2F;QAC3F,yFAAyF;QACzF,uFAAuF;QACvFA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEyB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAMhC,QAAQY,MAAM;QACpC,MAAMqB,WAAW,MAAMjC,QAAQY,MAAM;QACrC,MAAMsB,eAAexC,iBAAiBmB,KAAKd;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB5B,WAAWW;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUM,MAAMC,KAAKA,IAAIO,OAAO;QACnE,OAAO;YAAEO;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,OAAO;QAAE8B,OAAOnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;QAAGP,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","reconcile","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,8LAA8L;AAC9L,2JAA2J;AAC3J,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWjB,eAAee,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAElB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMmB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMX,QAAQW,MAAM,sBAAuB,MAAM,MAAMV,QAAQU,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMT,QAAQW,MAAM,gBAAiB,MAAM,MAAMV,QAAQU,MAAM,YAAYnB,iBAAiBoB,KAAKd;AACxG;AAEA,eAAemB,QAAQL,GAAmB;IACxC,MAAMM,WAAW3B,KAAKqB,IAAIO,OAAO,EAAE1B;IACnCH,UAAU4B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS9B,KAAK2B,UAAUV;IAE9B,gIAAgI;IAChI,qLAAqL;IACrL,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe3B,YAAY,MAAM2B;QACrC,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEyB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAM/B,QAAQW,MAAM;QACpC,MAAMqB,WAAW,MAAMhC,QAAQW,MAAM;QACrC,MAAMsB,eAAezC,iBAAiBoB,KAAKd;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB9B,WAAWa;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUM,MAAMC,KAAKA,IAAIO,OAAO;QACnE,OAAO;YAAEO;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,MAAM8B,QAAQnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;IAClF,0FAA0F;IAC1FzB,eAAeiB,MAAM+B;IACrB,OAAO;QAAEA,OAAOA;QAAO9B,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpE"}

package/dist/esm/store/duckdb/reconcile.js

@@ -5,14 +5,8 @@ import { listFiles, RESERVED_COLUMNS } from '../../scan/index.js';
 import { reparseFiles } from '../../scan/reparse.js';
 import { getColumns, quoteIdent } from '../shared.js';
 import { withTransaction } from '../transaction.js';
-// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per
-// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain
-// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from
-// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one
-// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared
-// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the
-// only DuckDB type that can hold the different JS types mapValue() produces across files for
-// the same key).
+// Mirrors sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per discovered key, ON CONFLICT upsert). `content` is
+// a plain table, not FTS-virtual, so rows are maintained here unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
 const CORE_FRONTMATTER_COLUMNS = new Set([
     'path',
     '_mtime',
@@ -31,9 +25,8 @@ function contentRow(doc) {
         doc.search.text
     ];
 }
-// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity
-// fence anyway so a runaway frontmatter generator fails with a clear message instead of an
-// unbounded ALTER TABLE loop.
+// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway
+// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.
 const MAX_FRONTMATTER_COLUMNS = 10000;
 export async function reconcile(conn, cfg, baseDir) {
     const files = listFiles(cfg, baseDir);

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per\n// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain\n// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from\n// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one\n// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared\n// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the\n// only DuckDB type that can hold the different JS types mapValue() produces across files for\n// the same key).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity\n// fence anyway so a runaway frontmatter generator fails with a clear message instead of an\n// unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,4FAA4F;AAC5F,8FAA8F;AAC9F,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iBAAiB;AACjB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,6FAA6F;AAC7F,2FAA2F;AAC3F,8BAA8B;AAC9B,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per discovered key, ON CONFLICT upsert). `content` is\n// a plain table, not FTS-virtual, so rows are maintained here unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,oIAAoI;AACpI,wOAAwO;AACxO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/duckdb/sql-functions.js

@@ -1,11 +1,8 @@
 import { segmentMatch } from '../../text/segment.js';
 import { basenameImpl, hasImpl } from '../sql-functions.js';
 import { duckdbApi } from './native.js';
-// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional
-// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a
-// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already
-// resolved the package, possibly via an on-demand install, and a second independent bare-specifier
-// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).
+// Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency (open.ts's connect()).
+// Goes through native.ts's shared duckdbApi(), not its own bare import, since a second independent lookup would hit Node's stale pre-install resolution miss.
 export async function registerFunctions(conn) {
     const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();
     conn.registerScalarFunction(DuckDBScalarFunction.create({
@@ -36,10 +33,8 @@ export async function registerFunctions(conn) {
             outputVector.flush();
         }
     }));
-    // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars
-    // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)
-    // and never calls this, but hand-written raw SQL naming it still gets the real implementation
-    // rather than a silent passthrough (PRINCIPLES: no-silent-modes).
+    // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars need (text/segment.ts); this store's own
+    // lexical index uses contains() instead and never calls this, but hand-written raw SQL naming it gets the real implementation, not a silent passthrough (PRINCIPLES: no-silent-modes).
     conn.registerScalarFunction(DuckDBScalarFunction.create({
         name: 'segment',
         parameterTypes: [

package/dist/esm/store/duckdb/sql-functions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional\n// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a\n// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already\n// resolved the package, possibly via an on-demand install, and a second independent bare-specifier\n// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars\n  // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)\n  // and never calls this, but hand-written raw SQL naming it still gets the real implementation\n  // rather than a silent passthrough (PRINCIPLES: no-silent-modes).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["segmentMatch","basenameImpl","hasImpl","duckdbApi","registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","undefined","textVec","String"],"mappings":"AACA,SAASA,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,EAAEC,OAAO,QAAQ,sBAAsB;AAC5D,SAASC,SAAS,QAAQ,cAAc;AAExC,iGAAiG;AACjG,gGAAgG;AAChG,6FAA6F;AAC7F,mGAAmG;AACnG,iGAAiG;AACjG,OAAO,eAAeC,kBAAkBC,IAAsB;IAC5D,MAAM,EAAEC,iBAAiB,EAAEC,oBAAoB,EAAEC,iBAAiB,EAAE,GAAG,MAAML;IAC7EE,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;YAAEL,kBAAkBK,QAAQ;SAAC;QACxEC,YAAYR,kBAAkBO,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMC,WAAWF,WAAWG,eAAe,CAAC;YAC5C,MAAMC,WAAWJ,WAAWG,eAAe,CAAC;YAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGpB,QAAQiB,SAASM,OAAO,CAACH,IAAID,SAASI,OAAO,CAACH;YACpHJ,aAAaQ,KAAK;QACpB;IACF;IAEFrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5Cc,aAAanB,kBAAkBK,QAAQ;QACvCC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMU,UAAUX,WAAWG,eAAe,CAAC;YAC3C,MAAMS,YAAYZ,WAAWa,WAAW,GAAG,IAAIb,WAAWG,eAAe,CAAC,KAAK;YAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGrB,aAAa2B,QAAQH,OAAO,CAACH,IAAIO,YAAYA,UAAUJ,OAAO,CAACH,KAAKS;YAC1Ib,aAAaQ,KAAK;QACpB;IACF;IAEF,0FAA0F;IAC1F,8FAA8F;IAC9F,8FAA8F;IAC9F,kEAAkE;IAClErB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5CC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gBAEgDc;YAD1F,MAAMA,UAAUf,WAAWG,eAAe,CAAC;YAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGtB,aAAaiC,QAAOD,mBAAAA,QAAQP,OAAO,CAACH,gBAAhBU,8BAAAA,mBAAsB;YAChHd,aAAaQ,KAAK;QACpB;IACF;AAEJ"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency (open.ts's connect()).\n// Goes through native.ts's shared duckdbApi(), not its own bare import, since a second independent lookup would hit Node's stale pre-install resolution miss.\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars need (text/segment.ts); this store's own\n  // lexical index uses contains() instead and never calls this, but hand-written raw SQL naming it gets the real implementation, not a silent passthrough (PRINCIPLES: no-silent-modes).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["segmentMatch","basenameImpl","hasImpl","duckdbApi","registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","undefined","textVec","String"],"mappings":"AACA,SAASA,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,EAAEC,OAAO,QAAQ,sBAAsB;AAC5D,SAASC,SAAS,QAAQ,cAAc;AAExC,mIAAmI;AACnI,8JAA8J;AAC9J,OAAO,eAAeC,kBAAkBC,IAAsB;IAC5D,MAAM,EAAEC,iBAAiB,EAAEC,oBAAoB,EAAEC,iBAAiB,EAAE,GAAG,MAAML;IAC7EE,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;YAAEL,kBAAkBK,QAAQ;SAAC;QACxEC,YAAYR,kBAAkBO,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMC,WAAWF,WAAWG,eAAe,CAAC;YAC5C,MAAMC,WAAWJ,WAAWG,eAAe,CAAC;YAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGpB,QAAQiB,SAASM,OAAO,CAACH,IAAID,SAASI,OAAO,CAACH;YACpHJ,aAAaQ,KAAK;QACpB;IACF;IAEFrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5Cc,aAAanB,kBAAkBK,QAAQ;QACvCC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMU,UAAUX,WAAWG,eAAe,CAAC;YAC3C,MAAMS,YAAYZ,WAAWa,WAAW,GAAG,IAAIb,WAAWG,eAAe,CAAC,KAAK;YAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGrB,aAAa2B,QAAQH,OAAO,CAACH,IAAIO,YAAYA,UAAUJ,OAAO,CAACH,KAAKS;YAC1Ib,aAAaQ,KAAK;QACpB;IACF;IAEF,mIAAmI;IACnI,uLAAuL;IACvLrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5CC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gBAEgDc;YAD1F,MAAMA,UAAUf,WAAWG,eAAe,CAAC;YAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGtB,aAAaiC,QAAOD,mBAAAA,QAAQP,OAAO,CAACH,gBAAhBU,8BAAAA,mBAAsB;YAChHd,aAAaQ,KAAK;QACpB;IACF;AAEJ"}

package/dist/esm/store/duckdb/store.js

@@ -7,22 +7,15 @@ import { fieldStats } from './fieldStats.js';
 import { createLexicalIndex } from './lexical.js';
 import { reconcile } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
-// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (D1: fts BM25 +
-// contains(), see lexical.ts), and vectors (D2: native FLOAT[N] + array_cosine_similarity, see
-// vectors.ts) are implemented. 'snippets' is declined: no snippet()-equivalent exists, so every
-// lexical hit relies on the caller's JS excerpt fallback (commands/search.ts) rather than a
-// bounded engine-native one. 'phrases' means quoted-phrase matching only (types.ts):
-// prefix/boolean/NEAR/column-filter FTS5 syntax is not claimed here -- lexical.ts rejects it
-// rather than answer it differently.
+// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native
+// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.
 export const CAPABILITIES = new Set([
     'lexical',
     'phrases',
     'vectors'
 ]);
-// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth
-// (see transaction.ts) is tracked against the same object everywhere. The instance is the
-// native database handle that owns the WAL: close() must close it, not just disconnect the
-// connection, or DuckDB never checkpoints and the next open reads a mismatched WAL.
+// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.
+// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.
 export function createStore(instance, duckdb, conn, cfg, baseDir) {
     const lex = createLexicalIndex(conn);
     return {
@@ -58,9 +51,8 @@ export function createStore(instance, duckdb, conn, cfg, baseDir) {
         lexical: {
             query: lex.query
         },
-        // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the
-        // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is
-        // still correct against a wider column.
+        // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --
+        // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.
         vectors: {
             pending: ()=>pendingRows(conn),
             writeVectors: (rows)=>writeVectorBatch(duckdb, conn, STORE_DIMS, rows),

package/dist/esm/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (D1: fts BM25 +\n// contains(), see lexical.ts), and vectors (D2: native FLOAT[N] + array_cosine_similarity, see\n// vectors.ts) are implemented. 'snippets' is declined: no snippet()-equivalent exists, so every\n// lexical hit relies on the caller's JS excerpt fallback (commands/search.ts) rather than a\n// bounded engine-native one. 'phrases' means quoted-phrase matching only (types.ts):\n// prefix/boolean/NEAR/column-filter FTS5 syntax is not claimed here -- lexical.ts rejects it\n// rather than answer it differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere. The instance is the\n// native database handle that owns the WAL: close() must close it, not just disconnect the\n// connection, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is\n    // still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,+FAA+F;AAC/F,gGAAgG;AAChG,4FAA4F;AAC5F,qFAAqF;AACrF,6FAA6F;AAC7F,qCAAqC;AACrC,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8FAA8F;AAC9F,0FAA0F;AAC1F,2FAA2F;AAC3F,oFAAoF;AACpF,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAMrB;YACJ,MAAMsB,SAAS,MAAMtB,UAAUS,MAAMC,KAAKC;YAC1C,uFAAuF;YACvF,4DAA4D;YAC5DC,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,sFAAsF;QACtF,0FAA0F;QAC1F,wCAAwC;QACxCC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMjB,YAAYwC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI1C,YAAY4C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMjB,YAAY+C,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8JAA8J;AAC9J,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAMrB;YACJ,MAAMsB,SAAS,MAAMtB,UAAUS,MAAMC,KAAKC;YAC1C,uFAAuF;YACvF,4DAA4D;YAC5DC,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMjB,YAAYwC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI1C,YAAY4C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMjB,YAAY+C,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}

package/dist/esm/store/duckdb/vectors.js

@@ -2,24 +2,15 @@ import { withTransaction } from '../transaction.js';
 import { asCosine, sampleEvenly } from '../vectors.js';
 import { rewriteUpdate } from './batch.js';
 import { duckdbApi } from './native.js';
-// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by
-// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with
-// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --
-// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own
-// representation and stays as it is. Every function here takes `dims` as a parameter rather
-// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real
-// constant at the call site).
-//
-// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only
-// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's
-// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever
-// open.ts already resolved, including via an on-demand install (see native.ts).
+// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and
+// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.
+// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency
+// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 const untyped = undefined;
-// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced
-// under the column's width); zero-padding leaves cosine scores unchanged since the added
-// dimensions contribute nothing to either vector's dot product or norm.
+// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);
+// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.
 function padded(values, dims) {
     const out = new Array(dims).fill(0);
     for(let d = 0; d < Math.min(values.length, dims); d++)out[d] = values[d];
@@ -30,19 +21,14 @@ function inClause(column, count) {
         length: count
     }, ()=>'?').join(', ')})`;
 }
-// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces
-// for every store, see toStore); dequantizing into a plain float array is this store's own
-// representation choice, so both stores are handed the same vectors and diverge only here.
+// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);
+// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.
 function dequantize(row, dims) {
     const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
     return padded(Array.from(q, (v)=>v * row.scale), dims);
 }
-// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's
-// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native
-// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --
-// automatic type inference on the raw values misreads an all-integer first component (a real
-// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row
-// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
+// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the
+// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
 export async function writeVectorBatch(duckdb, conn, dims, rows) {
     if (rows.length === 0) return;
     const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
@@ -64,9 +50,8 @@ export async function writeVectorBatch(duckdb, conn, dims, rows) {
         }
     });
 }
-// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks
-// the winning chunk's line range from the same row max(score) came from, per path. An empty
-// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.
+// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks the winning chunk's line range from the same row max(score) came from, per path.
+// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.
 export async function scanCandidates(duckdb, qv, dims, fetch, allowed) {
     if (allowed && allowed.size === 0) return [];
     const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
@@ -106,10 +91,8 @@ export async function scanCandidates(duckdb, qv, dims, fetch, allowed) {
         stmt.destroySync();
     }
 }
-// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed
-// into one native scan: the sampled target vectors become a small VALUES list, cross-joined
-// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost
-// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.
+// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become
+// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.
 export async function scanSimilar(duckdb, conn, dims, path, opts) {
     if (opts.allowed && opts.allowed.size === 0) return [];
     const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');

package/dist/esm/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by\n// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with\n// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --\n// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own\n// representation and stays as it is. Every function here takes `dims` as a parameter rather\n// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real\n// constant at the call site).\n//\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only\n// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's\n// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever\n// open.ts already resolved, including via an on-demand install (see native.ts).\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced\n// under the column's width); zero-padding leaves cosine scores unchanged since the added\n// dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces\n// for every store, see toStore); dequantizing into a plain float array is this store's own\n// representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's\n// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native\n// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --\n// automatic type inference on the raw values misreads an all-integer first component (a real\n// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row\n// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks\n// the winning chunk's line range from the same row max(score) came from, per path. An empty\n// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed\n// into one native scan: the sampled target vectors become a small VALUES list, cross-joined\n// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost\n// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","score","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,mFAAmF;AACnF,uFAAuF;AACvF,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAC5F,8FAA8F;AAC9F,8BAA8B;AAC9B,EAAE;AACF,4FAA4F;AAC5F,8FAA8F;AAC9F,gGAAgG;AAChG,gFAAgF;AAEhF,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,6FAA6F;AAC7F,yFAAyF;AACzF,wEAAwE;AACxE,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,2FAA2F;AAC3F,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,0FAA0F;AAC1F,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,6FAA6F;AAC7F,4FAA4F;AAC5F,2FAA2F;AAC3F,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;YAAC4B,WAAW7B,OAAO6C,IAAI3C;eAAW+C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ7B;eAAU+C,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC7FoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACjH,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,8FAA8F;AAC9F,wFAAwF;AACxF,OAAO,eAAeiB,YAAYlC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAEsB,IAAgE;IACxK,IAAIA,KAAKd,OAAO,IAAIc,KAAKd,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMc,aAAa,MAAMnC,KAAKQ,OAAO,CAAC;IACtC,MAAM4B,aAAc,MAAMD,WAAWE,GAAG,CAACzB;IACzC,IAAIwB,WAAWtD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUtE,aAAaoE,YAAYb,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMqE,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMlB,cAAcY,KAAKd,OAAO,GAAG;WAAIc,KAAKd,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE6B,QAAQf,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEoD,YAAYzD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYwD,YAAYzD,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEoD,KAAKd,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIgE,QAAQf,GAAG,CAAC,CAACkB,IAAMvC,WAAW7B,OAAOoE,GAAGlE;YAASqC;eAAS2B;eAAgBjB;YAAaY,KAAKQ,CAAC;SAAC;QAChI,MAAMhC,QAAsB;eAAI4B,QAAQf,GAAG,CAAC,IAAMpB,MAAMC,QAAQ7B;YAAQJ;eAAYoE,YAAYhB,GAAG,CAAC,IAAMpD;eAAamD,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACxE,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks the winning chunk's line range from the same row max(score) came from, per path.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","score","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,oIAAoI;AACpI,8QAA8Q;AAC9Q,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,8KAA8K;AAC9K,oGAAoG;AACpG,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;YAAC4B,WAAW7B,OAAO6C,IAAI3C;eAAW+C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ7B;eAAU+C,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC7FoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACjH,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeiB,YAAYlC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAEsB,IAAgE;IACxK,IAAIA,KAAKd,OAAO,IAAIc,KAAKd,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMc,aAAa,MAAMnC,KAAKQ,OAAO,CAAC;IACtC,MAAM4B,aAAc,MAAMD,WAAWE,GAAG,CAACzB;IACzC,IAAIwB,WAAWtD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwD,UAAUtE,aAAaoE,YAAYb,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMqE,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMlB,cAAcY,KAAKd,OAAO,GAAG;WAAIc,KAAKd,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE6B,QAAQf,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEoD,YAAYzD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYwD,YAAYzD,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEoD,KAAKd,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIgE,QAAQf,GAAG,CAAC,CAACkB,IAAMvC,WAAW7B,OAAOoE,GAAGlE;YAASqC;eAAS2B;eAAgBjB;YAAaY,KAAKQ,CAAC;SAAC;QAChI,MAAMhC,QAAsB;eAAI4B,QAAQf,GAAG,CAAC,IAAMpB,MAAMC,QAAQ7B;YAAQJ;eAAYoE,YAAYhB,GAAG,CAAC,IAAMpD;eAAamD,YAAYC,GAAG,CAAC,IAAMpD;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAYhE,SAAS4D,EAAEK,KAAK;YAAE,CAAA;IACxE,SAAU;QACRzB,KAAKS,WAAW;IAClB;AACF"}

package/dist/esm/store/index.d.ts

@@ -2,7 +2,8 @@ import type { ResolvedConfig } from '../config/index.js';
 import type { OpenResult } from './sqlite/open.js';
 export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
 export declare function requireWatchConcurrency(cfg: ResolvedConfig): void;
-export { getMeta, setMeta } from './meta.js';
+export { clearCache } from './cache.js';
+export { getMeta, setMeta } from './shared.js';
 export type { OpenResult } from './sqlite/open.js';
-export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';

package/dist/esm/store/index.js

@@ -32,10 +32,8 @@ function entryFor(name) {
 export async function openStore(cfg) {
     const name = storeName(cfg);
     const entry = entryFor(name);
-    // The one unambiguous, config-level capability need: an embed block some preset actually
-    // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level
-    // (every preset defaults to wanting "words"), so a store lacking 'lexical' fails loudly at
-    // first use instead (see duckdb/store.ts's lexical.query()).
+    // The one unambiguous, config-level capability need: an embed block some preset actually uses
+    // for vectors. Lexical search has no such signal at this level (every preset defaults to "words"), so a store lacking it fails loudly at first use.
     if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "vectors" in this build, but this config's "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)`);
     }
@@ -49,5 +47,6 @@ export function requireWatchConcurrency(cfg) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "watch-concurrency" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set "store" to "sqlite" in sense.config.json`);
     }
 }
-export { getMeta, setMeta } from './meta.js';
-export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { clearCache } from './cache.js';
+export { getMeta, setMeta } from './shared.js';
+export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';