sensemaking 0.24.6 → 0.24.7

package/dist/cjs/store/duckdb/open.js

@@ -33,6 +33,7 @@ var _nativets = require("./native.js");
 var _reconcilets = require("./reconcile.js");
 var _sqlfunctionsts = require("./sql-functions.js");
 var _storets = require("./store.js");
+var _vectorsts = require("./vectors.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -168,7 +169,7 @@ var DB_FILENAME = 'cache.duckdb';
 var SCHEMA_VERSION = '4';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
-function ensureSchema(_handle, conn, cfg) {
+function ensureSchema(handle, conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
         return _ts_generator(this, function(_state) {
@@ -203,21 +204,21 @@ function ensureSchema(_handle, conn, cfg) {
                 case 5:
                     _state.trys.push([
                         5,
-                        12,
                         13,
-                        14
+                        14,
+                        15
                     ]);
                     _iterator = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
                     _state.label = 6;
                 case 6:
                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
                         3,
-                        11
+                        12
                     ];
                     feature = _step.value;
                     if (!(feature.name === 'embed')) return [
                         3,
-                        8
+                        9
                     ];
                     return [
                         4,
@@ -226,37 +227,43 @@ function ensureSchema(_handle, conn, cfg) {
                 case 7:
                     _state.sent();
                     return [
-                        3,
-                        10
+                        4,
+                        (0, _vectorsts.createVectorWriteStage)(handle.duckdb, _typests.STORE_DIMS)
                     ];
                 case 8:
+                    _state.sent();
+                    return [
+                        3,
+                        11
+                    ];
+                case 9:
                     return [
                         4,
                         feature.schema(conn)
                     ];
-                case 9:
-                    _state.sent();
-                    _state.label = 10;
                 case 10:
+                    _state.sent();
+                    _state.label = 11;
+                case 11:
                     _iteratorNormalCompletion = true;
                     return [
                         3,
                         6
                     ];
-                case 11:
+                case 12:
                     return [
                         3,
-                        14
+                        15
                     ];
-                case 12:
+                case 13:
                     err = _state.sent();
                     _didIteratorError = true;
                     _iteratorError = err;
                     return [
                         3,
-                        14
+                        15
                     ];
-                case 13:
+                case 14:
                     try {
                         if (!_iteratorNormalCompletion && _iterator.return != null) {
                             _iterator.return();
@@ -269,41 +276,41 @@ function ensureSchema(_handle, conn, cfg) {
                     return [
                         7
                     ];
-                case 14:
+                case 15:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'schema_version')
                     ];
-                case 15:
+                case 16:
                     if (!(_state.sent() === null)) return [
                         3,
-                        17
+                        18
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'schema_version', SCHEMA_VERSION)
                     ];
-                case 16:
-                    _state.sent();
-                    _state.label = 17;
                 case 17:
+                    _state.sent();
+                    _state.label = 18;
+                case 18:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'features')
                     ];
-                case 18:
+                case 19:
                     if (!(_state.sent() === null)) return [
                         3,
-                        20
+                        21
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES))
                     ];
-                case 19:
-                    _state.sent();
-                    _state.label = 20;
                 case 20:
+                    _state.sent();
+                    _state.label = 21;
+                case 21:
                     return [
                         2
                     ];

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","createVectorWriteStage","duckdb","schema","getMeta","setMeta","featureSignature","FEATURES","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAiBaA;eAAAA;;QAGAC;eAAAA;;QAyEAC;eAAAA;;QAcSC;eAAAA;;;uBAzGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;yBACW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEhC,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;;YAKxE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;wBAAMC,IAAAA,iCAAsB,EAACR,OAAOS,MAAM,EAAEF,mBAAU;;;oBAAtD;oBACA;;;;;oBAEF;;wBAAMJ,QAAQO,MAAM,CAACT;;;oBAArB;;;oBARG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAUA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMf,MAAoB;;;YACvCA,OAAOS,MAAM,CAACO,cAAc;YAC5BhB,OAAOiB,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLd,QACAQ,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BV,SAAS;;;;;;oBACFc;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACrB;;;oBAAxB;oBACMR,OAAO8B,IAAAA,8BAAgB,EAACtB;oBAC9B;;wBAAO;4BAAET,QAAQ;gCAAEiB,UAAAA;gCAAUR,QAAAA;4BAAO;4BAAGR,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMR,MAAM;4BAAEE,UAAAA;4BAAUR,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMc;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAJ,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFqB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAActC,QAAQC;eAASqC,IAAAA,oBAAW,EAACtC,OAAOiB,QAAQ,EAAEjB,OAAOS,MAAM,EAAER;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}

package/dist/cjs/store/duckdb/vectors.d.cts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/cjs/store/duckdb/vectors.d.ts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/cjs/store/duckdb/vectors.js

@@ -9,6 +9,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get createVectorWriteStage () {
+        return createVectorWriteStage;
+    },
     get scanCandidates () {
         return scanCandidates;
     },
@@ -21,7 +24,6 @@ _export(exports, {
 });
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
-var _batchts = require("./batch.js");
 var _nativets = require("./native.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -182,6 +184,25 @@ function _unsupported_iterable_to_array(o, minLen) {
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 var untyped = undefined;
+var VECTOR_WRITE_STAGE = '_sense_vector_write_stage';
+function createVectorWriteStage(duckdb, dims) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        duckdb.run("CREATE TEMP TABLE ".concat(VECTOR_WRITE_STAGE, ' ("path" TEXT, chunk INTEGER, vector FLOAT[').concat(dims, "])"))
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 // The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);
 // zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.
 function padded(values, dims) {
@@ -206,7 +227,7 @@ function dequantize(row, dims) {
 }
 function writeVectorBatch(duckdb, conn, dims, rows) {
     return _async_to_generator(function() {
-        var _ref, arrayValue, ARRAY, DOUBLE, rewritten;
+        var _ref, ARRAY, FLOAT, vectorType;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -218,71 +239,62 @@ function writeVectorBatch(duckdb, conn, dims, rows) {
                         (0, _nativets.duckdbApi)()
                     ];
                 case 1:
-                    _ref = _state.sent(), arrayValue = _ref.arrayValue, ARRAY = _ref.ARRAY, DOUBLE = _ref.DOUBLE;
-                    rewritten = (0, _batchts.rewriteUpdate)('UPDATE embeddings SET vector = ? WHERE "path" = ? AND chunk = ?', rows.length);
-                    if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');
+                    _ref = _state.sent(), ARRAY = _ref.ARRAY, FLOAT = _ref.FLOAT;
+                    vectorType = ARRAY(FLOAT, dims);
                     return [
                         4,
                         (0, _transactionts.withTransaction)(conn, function() {
                             return _async_to_generator(function() {
-                                var stmt, values, types, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row;
+                                var appender, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row;
                                 return _ts_generator(this, function(_state) {
                                     switch(_state.label){
                                         case 0:
                                             return [
                                                 4,
-                                                duckdb.prepare(rewritten.sql)
+                                                duckdb.run("DELETE FROM ".concat(VECTOR_WRITE_STAGE))
                                             ];
                                         case 1:
-                                            stmt = _state.sent();
-                                            _state.label = 2;
-                                        case 2:
-                                            _state.trys.push([
-                                                2,
-                                                ,
+                                            _state.sent();
+                                            return [
                                                 4,
-                                                5
-                                            ]);
-                                            values = [];
-                                            types = [];
-                                            _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                                duckdb.createAppender(VECTOR_WRITE_STAGE)
+                                            ];
+                                        case 2:
+                                            appender = _state.sent();
                                             try {
-                                                for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                                                    row = _step.value;
-                                                    values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);
-                                                    types.push(ARRAY(DOUBLE, dims), untyped, untyped);
-                                                }
-                                            } catch (err) {
-                                                _didIteratorError = true;
-                                                _iteratorError = err;
-                                            } finally{
+                                                _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                                                 try {
-                                                    if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                                        _iterator.return();
+                                                    for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                                        row = _step.value;
+                                                        appender.appendVarchar(row.path);
+                                                        appender.appendInteger(row.chunk);
+                                                        appender.appendArray(dequantize(row, dims), vectorType);
+                                                        appender.endRow();
                                                     }
+                                                } catch (err) {
+                                                    _didIteratorError = true;
+                                                    _iteratorError = err;
                                                 } finally{
-                                                    if (_didIteratorError) {
-                                                        throw _iteratorError;
+                                                    try {
+                                                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                                            _iterator.return();
+                                                        }
+                                                    } finally{
+                                                        if (_didIteratorError) {
+                                                            throw _iteratorError;
+                                                        }
                                                     }
                                                 }
+                                                appender.flushSync();
+                                            } finally{
+                                                appender.closeSync();
                                             }
-                                            stmt.bind(values, types);
                                             return [
                                                 4,
-                                                stmt.run()
+                                                duckdb.run("UPDATE embeddings SET vector = staged.vector FROM ".concat(VECTOR_WRITE_STAGE, ' AS staged WHERE embeddings."path" = staged."path" AND embeddings.chunk = staged.chunk'))
                                             ];
                                         case 3:
                                             _state.sent();
-                                            return [
-                                                3,
-                                                5
-                                            ];
-                                        case 4:
-                                            stmt.destroySync();
-                                            return [
-                                                7
-                                            ];
-                                        case 5:
                                             return [
                                                 2
                                             ];

package/dist/cjs/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","duckdbApi","rewriteUpdate","Error","withTransaction","stmt","types","prepare","sql","push","path","chunk","bind","run","destroySync","qv","fetch","allowed","allowedList","query","queryNorm","score","reader","size","sqrt","reduce","sum","value","map","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkEsBA;eAAAA;;QAmCAC;eAAAA;;QA5DAC;eAAAA;;;6BAxCU;yBAEO;uBACT;wBACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,IAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,IAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCtB;AAEJ;AAIO,SAAeL,iBAAiB4B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEyB,IAAsB;;YAE/E,MAA9BC,YAAYC,OAAOC,QACrBC;;;;oBAFN,IAAIJ,KAAKlB,MAAM,KAAK,GAAG;;;oBACe;;wBAAMuB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBC,YAAYE,IAAAA,sBAAa,EAAC,mEAAmEN,KAAKlB,MAAM;oBAC9G,IAAI,CAACsB,WAAW,MAAM,IAAIG,MAAM;oBAEhC;;wBAAMC,IAAAA,8BAAe,EAACT,MAAM;;oCACpBU,MAEEnC,QACAoC,OACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB;;;;4CAJA;;gDAAMS,OAAOa,OAAO,CAACP,UAAUQ,GAAG;;;4CAAzCH,OAAO;;;;;;;;;4CAELnC;4CACAoC;4CACD,kCAAA,2BAAA;;gDAAL,IAAK,YAAaV,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;oDAAbX,MAAN;oDACHf,OAAOuC,IAAI,CAACZ,WAAWb,WAAWC,KAAKd,QAAQc,IAAIyB,IAAI,EAAEzB,IAAI0B,KAAK;oDAClEL,MAAMG,IAAI,CAACX,MAAMC,QAAQ5B,OAAOJ,SAASA;gDAC3C;;gDAHK;gDAAA;;;yDAAA,6BAAA;wDAAA;;;wDAAA;8DAAA;;;;4CAILsC,KAAKO,IAAI,CAAC1C,QAAQoC;4CAClB;;gDAAMD,KAAKQ,GAAG;;;4CAAd;;;;;;4CAEAR,KAAKS,WAAW;;;;;;;;;;4BAEpB;;;;oBAdA;;;;;;IAeF;;AAIO,SAAelD,eAAe8B,MAAwB,EAAEqB,EAAgB,EAAE5C,IAAY,EAAE6C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BpB,YAAYC,OAAOC,QACrBmB,aACAC,OACAC,WACAC,OACAb,KAaAH,MAEEnC,QACAoC,OAEAgB,QACA1B;;;;oBAzBR,IAAIqB,WAAWA,QAAQM,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMtB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBmB,cAAcD,UAAW,qBAAGA;oBAC5BE,QAAQlD,OAAO8C,IAAI5C;oBACnBiD,YAAY5C,KAAKgD,IAAI,CAACL,MAAMM,MAAM,CAAC,SAACC,KAAKC;+BAAUD,MAAMC,QAAQA;uBAAO;oBACxEN,QAAQD,cAAc,IAAI,QAAQ;oBAClCZ,MAAM,AAAC,4QAOoBS,OAFmBI,OAAM,yEAEsC,OAA/DJ,UAAU,AAAC,OAA6C,OAAvCtC,SAAS,UAAUuC,YAAYxC,MAAM,KAAM,IAAG;oBAMnF;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB,AAAC,qBAAIkD,cAAc;wBAAUvB,WAAWsB;8BAAU,qBAAGD;wBAAaF;;oBAC1FV,QAAsB,AAAC,qBAAIc,cAAc;wBAAUtB,MAAMC,QAAQ5B;8BAAS,qBAAG+C,YAAYU,GAAG,CAAC;+BAAM7D;;wBAAUA;;oBACnHsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKwB,aAAa;;;oBAAjCP,SAAS;oBACT1B,OAAO0B,OAAOQ,eAAe;oBACnC;;wBAAOlC,KAAKgC,GAAG,CAAC,SAACG;mCAAO;gCAAErB,MAAMqB,EAAErB,IAAI;gCAAEsB,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEV,KAAK;4BAAE;;;;oBAE/GhB,KAAKS,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAejD,YAAY6B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEuC,IAAY,EAAE2B,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9B3C,YAAYC,OAAOC,QACrB0C,aACAvB,aACAV,KAWAH,MAEEnC,QACAoC,OAEAgB,QACA1B;;;;oBA1BR,IAAIyC,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACM,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAM5B,KAAKY,OAAO,CAAC;;;oBAAhC+B,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAChC;;;oBAAnC6B,aAAc;oBACpB,IAAIA,WAAW7D,MAAM,KAAK,GAAG;;;;oBACvB8D,UAAUG,IAAAA,uBAAY,EAACJ,YAAYX,GAAG,CAAC,SAAC3C;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMa,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrB0C,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9B1B,cAAcmB,KAAKpB,OAAO,GAAI,qBAAGoB,KAAKpB,OAAO;oBAC7CT,MAAM,AAAC,+BAIPiC,OAJqCD,QAAQZ,GAAG,CAAC;+BAAM;uBAAO7C,IAAI,CAAC,OAAM,kTAKzEsD,OADAI,YAAY/D,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAY8D,YAAY/D,MAAM,KAAM,IAAG,YACd,OAAtE2D,KAAKpB,OAAO,GAAG,AAAC,OAA+C,OAAzCtC,SAAS,YAAYuC,YAAYxC,MAAM,KAAM,IAAG;oBAM/D;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB,AAAC,qBAAGsE,QAAQZ,GAAG,CAAC,SAACiB;+BAAMhD,WAAW5B,OAAO4E,GAAG1E;;wBAASuC;uBAAM,qBAAG+B,cAAa,qBAAGvB;wBAAamB,KAAKS,CAAC;;oBACzHxC,QAAsB,AAAC,qBAAGkC,QAAQZ,GAAG,CAAC;+BAAM9B,MAAMC,QAAQ5B;;wBAAQJ;uBAAS,qBAAG0E,YAAYb,GAAG,CAAC;+BAAM7D;yBAAU,qBAAGmD,YAAYU,GAAG,CAAC;+BAAM7D;;wBAAUA;;oBACvJsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKwB,aAAa;;;oBAAjCP,SAAS;oBACT1B,OAAO0B,OAAOQ,eAAe;oBACnC;;wBAAOlC,KAAKgC,GAAG,CAAC,SAACG;mCAAO;gCAAErB,MAAMqB,EAAErB,IAAI;gCAAEyB,YAAYC,IAAAA,mBAAQ,EAACL,EAAEV,KAAK;4BAAE;;;;oBAEtEhB,KAAKS,WAAW;;;;;;;;;;IAEpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\nconst VECTOR_WRITE_STAGE = '_sense_vector_write_stage';\n\n// Created once per native connection outside write transactions; its lifetime follows that connection.\nexport async function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void> {\n  await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} (\"path\" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// The native appender needs an explicit FLOAT array type: inference from an integer first component\n// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { ARRAY, FLOAT } = await duckdbApi();\n  const vectorType = ARRAY(FLOAT, dims);\n\n  await withTransaction(conn, async () => {\n    await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);\n    const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);\n    try {\n      for (const row of rows) {\n        appender.appendVarchar(row.path);\n        appender.appendInteger(row.chunk);\n        appender.appendArray(dequantize(row, dims), vectorType);\n        appender.endRow();\n      }\n      appender.flushSync();\n    } finally {\n      appender.closeSync();\n    }\n    await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings.\"path\" = staged.\"path\" AND embeddings.chunk = staged.chunk`);\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["createVectorWriteStage","scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","VECTOR_WRITE_STAGE","duckdb","dims","run","padded","values","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","conn","rows","ARRAY","FLOAT","vectorType","duckdbApi","withTransaction","appender","createAppender","appendVarchar","path","appendInteger","chunk","appendArray","endRow","flushSync","closeSync","qv","fetch","allowed","arrayValue","DOUBLE","allowedList","query","queryNorm","score","sql","stmt","types","reader","size","sqrt","reduce","sum","value","prepare","map","bind","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","destroySync","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkBsBA;eAAAA;;QAqDAC;eAAAA;;QAmCAC;eAAAA;;QA5DAC;eAAAA;;;6BA7CU;yBAEO;wBACb;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAChB,IAAMC,qBAAqB;AAGpB,SAAeN,uBAAuBO,MAAwB,EAAEC,IAAY;;;;;oBACjF;;wBAAMD,OAAOE,GAAG,CAAC,AAAC,qBAAoFD,OAAhEF,oBAAmB,+CAAkD,OAALE,MAAK;;;oBAA3G;;;;;;IACF;;AAEA,wHAAwH;AACxH,oIAAoI;AACpI,SAASE,OAAOC,MAAyB,EAAEH,IAAY;IACrD,IAAMI,MAAM,IAAIC,MAAcL,MAAMM,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEV,OAAOO,IAAKH,GAAG,CAACG,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,IAAMkB,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOrB,OACLG,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCzB;AAEJ;AAIO,SAAeL,iBAAiBI,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAE2B,IAAsB;;YAE5F,MAAjBC,OAAOC,OACTC;;;;oBAFN,IAAIH,KAAKjB,MAAM,KAAK,GAAG;;;oBACE;;wBAAMqB,IAAAA,mBAAS;;;oBAAf,OAAA,eAAjBH,QAAiB,KAAjBA,OAAOC,QAAU,KAAVA;oBACTC,aAAaF,MAAMC,OAAO7B;oBAEhC;;wBAAMgC,IAAAA,8BAAe,EAACN,MAAM;;oCAEpBO,UAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMhB;;;;4CAHb;;gDAAMlB,OAAOE,GAAG,CAAC,AAAC,eAAiC,OAAnBH;;;4CAAhC;4CACiB;;gDAAMC,OAAOmC,cAAc,CAACpC;;;4CAAvCmC,WAAW;4CACjB,IAAI;gDACG,kCAAA,2BAAA;;oDAAL,IAAK,YAAaN,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;wDAAbV,MAAN;wDACHgB,SAASE,aAAa,CAAClB,IAAImB,IAAI;wDAC/BH,SAASI,aAAa,CAACpB,IAAIqB,KAAK;wDAChCL,SAASM,WAAW,CAACvB,WAAWC,KAAKjB,OAAO8B;wDAC5CG,SAASO,MAAM;oDACjB;;oDALK;oDAAA;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;gDAMLP,SAASQ,SAAS;4CACpB,SAAU;gDACRR,SAASS,SAAS;4CACpB;4CACA;;gDAAM3C,OAAOE,GAAG,CAAC,AAAC,qDAAuE,OAAnBH,oBAAmB;;;4CAAzF;;;;;;4BACF;;;;oBAfA;;;;;;IAgBF;;AAIO,SAAeL,eAAeM,MAAwB,EAAE4C,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BC,YAAYlB,OAAOmB,QACrBC,aACAC,OACAC,WACAC,OACAC,KAaAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBAzBR,IAAIkB,WAAWA,QAAQW,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMzB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrBC,cAAcH,UAAW,qBAAGA;oBAC5BI,QAAQ/C,OAAOyC,IAAI3C;oBACnBkD,YAAY1C,KAAKiD,IAAI,CAACR,MAAMS,MAAM,CAAC,SAACC,KAAKC;+BAAUD,MAAMC,QAAQA;uBAAO;oBACxET,QAAQD,cAAc,IAAI,QAAQ;oBAClCE,MAAM,AAAC,4QAOoBP,OAFmBM,OAAM,yEAEsC,OAA/DN,UAAU,AAAC,OAA6C,OAAvClC,SAAS,UAAUqC,YAAYtC,MAAM,KAAM,IAAG;oBAMnF;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAI+C,cAAc;wBAAUJ,WAAWG;8BAAU,qBAAGD;wBAAaJ;;oBAC1FU,QAAsB,AAAC,qBAAIJ,cAAc;wBAAUtB,MAAMmB,QAAQ/C;8BAAS,qBAAGgD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACnHyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAE+B,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAE/GE,KAAKmB,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAe9E,YAAYK,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAEoC,IAAY,EAAEqC,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9B9B,YAAYlB,OAAOmB,QACrB8B,aACA7B,aACAI,KAWAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBA1BR,IAAI8C,KAAK5B,OAAO,IAAI4B,KAAK5B,OAAO,CAACW,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAM9B,KAAKmC,OAAO,CAAC;;;oBAAhCa,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAC1C;;;oBAAnCuC,aAAc;oBACpB,IAAIA,WAAWjE,MAAM,KAAK,GAAG;;;;oBACvBkE,UAAUG,IAAAA,uBAAY,EAACJ,YAAYb,GAAG,CAAC,SAAC7C;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMW,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrB8B,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9BhC,cAAcyB,KAAK5B,OAAO,GAAI,qBAAG4B,KAAK5B,OAAO;oBAC7CO,MAAM,AAAC,+BAIPyB,OAJqCD,QAAQd,GAAG,CAAC;+BAAM;uBAAO/C,IAAI,CAAC,OAAM,kTAKzE0D,OADAI,YAAYnE,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAYkE,YAAYnE,MAAM,KAAM,IAAG,YACd,OAAtE+D,KAAK5B,OAAO,GAAG,AAAC,OAA+C,OAAzClC,SAAS,YAAYqC,YAAYtC,MAAM,KAAM,IAAG;oBAM/D;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAGyE,QAAQd,GAAG,CAAC,SAACmB;+BAAMnC,WAAW5C,OAAO+E,GAAGjF;;wBAASoC;uBAAM,qBAAGyC,cAAa,qBAAG7B;wBAAayB,KAAKS,CAAC;;oBACzH5B,QAAsB,AAAC,qBAAGsB,QAAQd,GAAG,CAAC;+BAAMlC,MAAMmB,QAAQ/C;;wBAAQJ;uBAAS,qBAAGiF,YAAYf,GAAG,CAAC;+BAAMlE;yBAAU,qBAAGoD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACvJyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAEkC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAEtEE,KAAKmB,WAAW;;;;;;;;;;IAEpB"}

package/dist/esm/store/duckdb/open.js

@@ -9,13 +9,14 @@ import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
 import { duckdbDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
+import { createVectorWriteStage } from './vectors.js';
 export const DB_FILENAME = 'cache.duckdb';
 // Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
 // The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
 export const SCHEMA_VERSION = '4';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
-async function ensureSchema(_handle, conn, cfg) {
+async function ensureSchema(handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
@@ -25,6 +26,7 @@ async function ensureSchema(_handle, conn, cfg) {
         // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.
         if (feature.name === 'embed') {
             await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY ("path", chunk))`);
+            await createVectorWriteStage(handle.duckdb, STORE_DIMS);
             continue;
         }
         await feature.schema(conn);

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureSchema","handle","conn","cfg","exec","feature","name","duckdb","schema","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;IAC7E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWnB,eAAeiB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEpB,WAAW,+BAA+B,CAAC;YAChM,MAAMa,uBAAuBI,OAAOM,MAAM,EAAEvB;YAC5C;QACF;QACA,MAAMqB,QAAQG,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,iFAAiF;AACjF,eAAesB,MAAMR,MAAoB;IACvCA,OAAOM,MAAM,CAACG,cAAc;IAC5BT,OAAOU,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIX;IACJ,IAAII;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFb,SAAS,MAAMI,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBY;QACxB,MAAML,OAAOX,iBAAiBgB;QAC9B,OAAO;YAAEN,QAAQ;gBAAEU;gBAAUJ;YAAO;YAAGL;QAAK;IAC9C,EAAE,OAAOe,KAAK;QACZ,MAAMR,MAAM;YAAEE;YAAUJ;QAAO;QAC/B,MAAMU;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUxB;IACVyB,eAAexB;IACfyB,kBAAkB9B;IAClBmB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFgB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FlB;IACAJ,aAAa,CAACK,QAAQC,OAASN,YAAYK,OAAOU,QAAQ,EAAEV,OAAOM,MAAM,EAAEL;AAC7E,EAAE;AAEF,OAAO,eAAeyB,WAAWxB,GAAmB;IAClD,OAAOf,gBAAgBe,KAAKkB;AAC9B"}

package/dist/esm/store/duckdb/vectors.d.ts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/esm/store/duckdb/vectors.js

@@ -1,6 +1,5 @@
 import { withTransaction } from '../transaction.js';
 import { asCosine, sampleEvenly } from '../vectors.js';
-import { rewriteUpdate } from './batch.js';
 import { duckdbApi } from './native.js';
 // This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and
 // scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.
@@ -9,6 +8,11 @@ import { duckdbApi } from './native.js';
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 const untyped = undefined;
+const VECTOR_WRITE_STAGE = '_sense_vector_write_stage';
+// Created once per native connection outside write transactions; its lifetime follows that connection.
+export async function createVectorWriteStage(duckdb, dims) {
+    await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} ("path" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);
+}
 // The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);
 // zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.
 function padded(values, dims) {
@@ -27,27 +31,27 @@ function dequantize(row, dims) {
     const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
     return padded(Array.from(q, (v)=>v * row.scale), dims);
 }
-// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the
-// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
+// The native appender needs an explicit FLOAT array type: inference from an integer first component
+// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.
 export async function writeVectorBatch(duckdb, conn, dims, rows) {
     if (rows.length === 0) return;
-    const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
-    const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE "path" = ? AND chunk = ?', rows.length);
-    if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');
+    const { ARRAY, FLOAT } = await duckdbApi();
+    const vectorType = ARRAY(FLOAT, dims);
     await withTransaction(conn, async ()=>{
-        const stmt = await duckdb.prepare(rewritten.sql);
+        await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);
+        const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);
         try {
-            const values = [];
-            const types = [];
             for (const row of rows){
-                values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);
-                types.push(ARRAY(DOUBLE, dims), untyped, untyped);
+                appender.appendVarchar(row.path);
+                appender.appendInteger(row.chunk);
+                appender.appendArray(dequantize(row, dims), vectorType);
+                appender.endRow();
             }
-            stmt.bind(values, types);
-            await stmt.run();
+            appender.flushSync();
         } finally{
-            stmt.destroySync();
+            appender.closeSync();
         }
+        await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings."path" = staged."path" AND embeddings.chunk = staged.chunk`);
     });
 }
 // Best chunk per file by cosine, with the earliest chunk winning exact score ties.

package/dist/esm/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","query","queryNorm","sqrt","reduce","sum","value","score","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,oIAAoI;AACpI,8QAA8Q;AAC9Q,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,mFAAmF;AACnF,oGAAoG;AACpG,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMG,QAAQlD,OAAO6C,IAAI3C;IACzB,MAAMiD,YAAY5C,KAAK6C,IAAI,CAACF,MAAMG,MAAM,CAAC,CAACC,KAAKC,QAAUD,MAAMC,QAAQA,OAAO;IAC9E,MAAMC,QAAQL,cAAc,IAAI,QAAQ;IACxC,MAAMf,MAAM,CAAC;;;;;oDAKqC,EAAEoB,MAAM;;iCAE3B,EAAET,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKvF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAKkD,cAAc,IAAI,EAAE,GAAG;gBAACtB,WAAWqB;aAAO;eAAMD;YAAaH;SAAM;QACtG,MAAMT,QAAsB;eAAKc,cAAc,IAAI,EAAE,GAAG;gBAACrB,MAAMC,QAAQ7B;aAAM;eAAM+C,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC3HoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAEuB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACjH,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeuB,YAAYxC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAE4B,IAAgE;IACxK,IAAIA,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMoB,aAAa,MAAMzC,KAAKQ,OAAO,CAAC;IACtC,MAAMkC,aAAc,MAAMD,WAAWE,GAAG,CAAC/B;IACzC,IAAI8B,WAAW5D,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAM8D,UAAU5E,aAAa0E,YAAYZ,GAAG,CAAC,CAACzC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAM2E,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMxB,cAAckB,KAAKpB,OAAO,GAAG;WAAIoB,KAAKpB,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAEmC,QAAQd,GAAG,CAAC,IAAM,OAAO3C,IAAI,CAAC,MAAM;;;;MAI3E,EAAE0D,YAAY/D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAY8D,YAAY/D,MAAM,GAAG,GAAG,GAAG;MACtF,EAAE0D,KAAKpB,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIsE,QAAQd,GAAG,CAAC,CAACiB,IAAM7C,WAAW7B,OAAO0E,GAAGxE;YAASqC;eAASiC;eAAgBvB;YAAakB,KAAKQ,CAAC;SAAC;QAChI,MAAMtC,QAAsB;eAAIkC,QAAQd,GAAG,CAAC,IAAM3B,MAAMC,QAAQ7B;YAAQJ;eAAY0E,YAAYf,GAAG,CAAC,IAAM3D;eAAamD,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAE0B,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACxE,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\nconst VECTOR_WRITE_STAGE = '_sense_vector_write_stage';\n\n// Created once per native connection outside write transactions; its lifetime follows that connection.\nexport async function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void> {\n  await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} (\"path\" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// The native appender needs an explicit FLOAT array type: inference from an integer first component\n// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { ARRAY, FLOAT } = await duckdbApi();\n  const vectorType = ARRAY(FLOAT, dims);\n\n  await withTransaction(conn, async () => {\n    await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);\n    const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);\n    try {\n      for (const row of rows) {\n        appender.appendVarchar(row.path);\n        appender.appendInteger(row.chunk);\n        appender.appendArray(dequantize(row, dims), vectorType);\n        appender.endRow();\n      }\n      appender.flushSync();\n    } finally {\n      appender.closeSync();\n    }\n    await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings.\"path\" = staged.\"path\" AND embeddings.chunk = staged.chunk`);\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","duckdbApi","untyped","undefined","VECTOR_WRITE_STAGE","createVectorWriteStage","duckdb","dims","run","padded","values","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","conn","rows","ARRAY","FLOAT","vectorType","appender","createAppender","appendVarchar","path","appendInteger","chunk","appendArray","endRow","flushSync","closeSync","scanCandidates","qv","fetch","allowed","size","arrayValue","DOUBLE","allowedList","query","queryNorm","sqrt","reduce","sum","value","score","sql","stmt","prepare","types","map","bind","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","destroySync","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAChB,MAAMC,qBAAqB;AAE3B,uGAAuG;AACvG,OAAO,eAAeC,uBAAuBC,MAAwB,EAAEC,IAAY;IACjF,MAAMD,OAAOE,GAAG,CAAC,CAAC,kBAAkB,EAAEJ,mBAAmB,2CAA2C,EAAEG,KAAK,EAAE,CAAC;AAChH;AAEA,wHAAwH;AACxH,oIAAoI;AACpI,SAASE,OAAOC,MAAyB,EAAEH,IAAY;IACrD,MAAMI,MAAM,IAAIC,MAAcL,MAAMM,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEV,OAAOO,IAAKH,GAAG,CAACG,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,MAAMkB,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOrB,OACLG,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCzB;AAEJ;AAEA,oGAAoG;AACpG,gHAAgH;AAChH,OAAO,eAAe0B,iBAAiB3B,MAAwB,EAAE4B,IAAgB,EAAE3B,IAAY,EAAE4B,IAAsB;IACrH,IAAIA,KAAKlB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEmB,KAAK,EAAEC,KAAK,EAAE,GAAG,MAAMpC;IAC/B,MAAMqC,aAAaF,MAAMC,OAAO9B;IAEhC,MAAMT,gBAAgBoC,MAAM;QAC1B,MAAM5B,OAAOE,GAAG,CAAC,CAAC,YAAY,EAAEJ,oBAAoB;QACpD,MAAMmC,WAAW,MAAMjC,OAAOkC,cAAc,CAACpC;QAC7C,IAAI;YACF,KAAK,MAAMoB,OAAOW,KAAM;gBACtBI,SAASE,aAAa,CAACjB,IAAIkB,IAAI;gBAC/BH,SAASI,aAAa,CAACnB,IAAIoB,KAAK;gBAChCL,SAASM,WAAW,CAACtB,WAAWC,KAAKjB,OAAO+B;gBAC5CC,SAASO,MAAM;YACjB;YACAP,SAASQ,SAAS;QACpB,SAAU;YACRR,SAASS,SAAS;QACpB;QACA,MAAM1C,OAAOE,GAAG,CAAC,CAAC,kDAAkD,EAAEJ,mBAAmB,sFAAsF,CAAC;IAClL;AACF;AAEA,mFAAmF;AACnF,oGAAoG;AACpG,OAAO,eAAe6C,eAAe3C,MAAwB,EAAE4C,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEC,UAAU,EAAElB,KAAK,EAAEmB,MAAM,EAAE,GAAG,MAAMtD;IAC5C,MAAMuD,cAAcJ,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMK,QAAQhD,OAAOyC,IAAI3C;IACzB,MAAMmD,YAAY3C,KAAK4C,IAAI,CAACF,MAAMG,MAAM,CAAC,CAACC,KAAKC,QAAUD,MAAMC,QAAQA,OAAO;IAC9E,MAAMC,QAAQL,cAAc,IAAI,QAAQ;IACxC,MAAMM,MAAM,CAAC;;;;;oDAKqC,EAAED,MAAM;;iCAE3B,EAAEX,UAAU,CAAC,IAAI,EAAElC,SAAS,UAAUsC,YAAYvC,MAAM,GAAG,GAAG,GAAG;;;;;WAKvF,CAAC;IACV,MAAMgD,OAAO,MAAM3D,OAAO4D,OAAO,CAACF;IAClC,IAAI;QACF,MAAMtD,SAAwB;eAAKgD,cAAc,IAAI,EAAE,GAAG;gBAACJ,WAAWG;aAAO;eAAMD;YAAaL;SAAM;QACtG,MAAMgB,QAAsB;eAAKT,cAAc,IAAI,EAAE,GAAG;gBAACtB,MAAMmB,QAAQhD;aAAM;eAAMiD,YAAYY,GAAG,CAAC,IAAMlE;YAAUA;SAAQ;QAC3H+D,KAAKI,IAAI,CAAC3D,QAAQyD;QAClB,MAAMG,SAAS,MAAML,KAAKM,aAAa;QACvC,MAAMpC,OAAOmC,OAAOE,eAAe;QACnC,OAAOrC,KAAKiC,GAAG,CAAC,CAACK,IAAO,CAAA;gBAAE/B,MAAM+B,EAAE/B,IAAI;gBAAEgC,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAY9E,SAAS0E,EAAEV,KAAK;YAAE,CAAA;IACjH,SAAU;QACRE,KAAKa,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeC,YAAYzE,MAAwB,EAAE4B,IAAgB,EAAE3B,IAAY,EAAEmC,IAAY,EAAEsC,IAAgE;IACxK,IAAIA,KAAK5B,OAAO,IAAI4B,KAAK5B,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAM4B,aAAa,MAAM/C,KAAKgC,OAAO,CAAC;IACtC,MAAMgB,aAAc,MAAMD,WAAWE,GAAG,CAACzC;IACzC,IAAIwC,WAAWjE,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMmE,UAAUpF,aAAakF,YAAYd,GAAG,CAAC,CAAC5C,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAE2B,UAAU,EAAElB,KAAK,EAAEmB,MAAM,EAAE,GAAG,MAAMtD;IAC5C,MAAMoF,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAM9B,cAAcwB,KAAK5B,OAAO,GAAG;WAAI4B,KAAK5B,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMY,MAAM,CAAC,4BAA4B,EAAEoB,QAAQhB,GAAG,CAAC,IAAM,OAAO9C,IAAI,CAAC,MAAM;;;;MAI3E,EAAE+D,YAAYpE,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYmE,YAAYpE,MAAM,GAAG,GAAG,GAAG;MACtF,EAAE+D,KAAK5B,OAAO,GAAG,CAAC,IAAI,EAAElC,SAAS,YAAYsC,YAAYvC,MAAM,GAAG,GAAG,GAAG;;;;;WAKnE,CAAC;IACV,MAAMgD,OAAO,MAAM3D,OAAO4D,OAAO,CAACF;IAClC,IAAI;QACF,MAAMtD,SAAwB;eAAI0E,QAAQhB,GAAG,CAAC,CAACmB,IAAMjC,WAAW7C,OAAO8E,GAAGhF;YAASmC;eAAS2C;eAAgB7B;YAAawB,KAAKQ,CAAC;SAAC;QAChI,MAAMrB,QAAsB;eAAIiB,QAAQhB,GAAG,CAAC,IAAMhC,MAAMmB,QAAQhD;YAAQL;eAAYmF,YAAYjB,GAAG,CAAC,IAAMlE;eAAasD,YAAYY,GAAG,CAAC,IAAMlE;YAAUA;SAAQ;QAC/J+D,KAAKI,IAAI,CAAC3D,QAAQyD;QAClB,MAAMG,SAAS,MAAML,KAAKM,aAAa;QACvC,MAAMpC,OAAOmC,OAAOE,eAAe;QACnC,OAAOrC,KAAKiC,GAAG,CAAC,CAACK,IAAO,CAAA;gBAAE/B,MAAM+B,EAAE/B,IAAI;gBAAEmC,YAAY9E,SAAS0E,EAAEV,KAAK;YAAE,CAAA;IACxE,SAAU;QACRE,KAAKa,WAAW;IAClB;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.24.6",
+  "version": "0.24.7",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step.",
   "keywords": [
     "markdown",

package/skills/sense-setup/references/store-benchmarks.md

@@ -1,15 +1,15 @@
-<!-- sense-store-benchmark release=0.24.6 -->
+<!-- sense-store-benchmark release=0.24.7 -->
 # Current store benchmark summary
 
-The release assessment generated this file for store selection. Release `0.24.6` passed on 2026-09-16, measured on Apple M4 Pro with Node v26.7.0.
+The release assessment generated this file for store selection. Release `0.24.7` passed on 2026-09-17, measured on Apple M4 Pro with Node v26.7.0.
 
 The timing rows ran on the same 6,566-note tree. They include CLI startup and each store's complete selected path. Ranked candidates and downstream work can differ by store, so these are current operating measurements rather than an isolated database-engine contest.
 
 | Store | Cold index | Warm count | Lexical search | Semantic search | Portable semantic nDCG@10 |
 |---|---|---|---|---|---|
-| sqlite | 1,471 ms | 143 ms | 192 ms | 318 ms | 0.3306 |
-| duckdb | 2,149 ms | 162 ms | 279 ms | 399 ms | 0.3284 |
-| turso | 2,727 ms | 138 ms | 237 ms | 584 ms | 0.3307 |
+| sqlite | 1,641 ms | 156 ms | 210 ms | 349 ms | 0.3306 |
+| duckdb | 3,376 ms | 181 ms | 565 ms | 556 ms | 0.3284 |
+| turso | 3,704 ms | 154 ms | 296 ms | 664 ms | 0.3307 |
 
 Cold index is the first `status` that builds the cache. Warm count is a no-change `COUNT(*)` query. Lexical and semantic search are steady-state `sense search` commands. Lower timing is faster. Higher nDCG@10 is better; that quality column uses the same NFCorpus queries, judgments, result count, and model on every store.