sensemaking 0.24.5 → 0.24.7

package/dist/cjs/lib/watch-notification.d.cts

@@ -0,0 +1 @@
+export declare function shouldReconcileWatchNotification(filename: string | Buffer | null): boolean;

package/dist/cjs/lib/watch-notification.d.ts

@@ -0,0 +1 @@
+export declare function shouldReconcileWatchNotification(filename: string | Buffer | null): boolean;

package/dist/cjs/lib/watch-notification.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "shouldReconcileWatchNotification", {
+    enumerable: true,
+    get: function() {
+        return shouldReconcileWatchNotification;
+    }
+});
+var _typests = require("../config/types.js");
+function shouldReconcileWatchNotification(filename) {
+    return typeof filename !== 'string' || !filename.startsWith(_typests.STATE_DIR);
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/lib/watch-notification.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/lib/watch-notification.ts"],"sourcesContent":["import { STATE_DIR } from '../config/types.ts';\n\nexport function shouldReconcileWatchNotification(filename: string | Buffer | null): boolean {\n  return typeof filename !== 'string' || !filename.startsWith(STATE_DIR);\n}\n"],"names":["shouldReconcileWatchNotification","filename","startsWith","STATE_DIR"],"mappings":";;;;+BAEgBA;;;eAAAA;;;uBAFU;AAEnB,SAASA,iCAAiCC,QAAgC;IAC/E,OAAO,OAAOA,aAAa,YAAY,CAACA,SAASC,UAAU,CAACC,kBAAS;AACvE"}

package/dist/cjs/store/duckdb/native.d.cts

@@ -1,4 +1,4 @@
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
-export declare const DUCKDB_VERSION = "1.5.5-r.4";
-export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.4";
+export declare const DUCKDB_VERSION = "1.5.5-r.5";
+export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.5";
 export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/cjs/store/duckdb/native.d.ts

@@ -1,4 +1,4 @@
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
-export declare const DUCKDB_VERSION = "1.5.5-r.4";
-export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.4";
+export declare const DUCKDB_VERSION = "1.5.5-r.5";
+export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.5";
 export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/cjs/store/duckdb/native.js

@@ -24,7 +24,7 @@ _export(exports, {
 });
 var _nativets = require("../native.js");
 var DUCKDB_PACKAGE = '@duckdb/node-api';
-var DUCKDB_VERSION = '1.5.5-r.4';
+var DUCKDB_VERSION = '1.5.5-r.5';
 var DUCKDB_INSTALL_SPEC = "".concat(DUCKDB_PACKAGE, "@").concat(DUCKDB_VERSION);
 var DUCKDB = {
     store: 'duckdb',

package/dist/cjs/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_VERSION = '1.5.5-r.4';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  version: DUCKDB_VERSION,\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["DUCKDB_INSTALL_SPEC","DUCKDB_PACKAGE","DUCKDB_VERSION","duckdbApi","DUCKDB","store","pkg","sizeHint","version","installSpec","duckdbApiPromise","loadOrInstall","packageNodeModules"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QAFAC;eAAAA;;QACAC;eAAAA;;QAeGC;eAAAA;;;wBAlByD;AAElE,IAAMF,iBAAiB;AACvB,IAAMC,iBAAiB;AACvB,IAAMF,sBAAsB,AAAC,GAAoBE,OAAlBD,gBAAe,KAAkB,OAAfC;AAExD,IAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKL;IACLM,UAAU;IACVC,SAASN;IACTO,aAAaT;AACf;AAEA,IAAIU;AAIG,SAASP;IACd,IAAI,CAACO,kBAAkBA,mBAAmBC,IAAAA,uBAAa,EAAoCP,QAAQQ,IAAAA,4BAAkB;IACrH,OAAOF;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_VERSION = '1.5.5-r.5';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  version: DUCKDB_VERSION,\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["DUCKDB_INSTALL_SPEC","DUCKDB_PACKAGE","DUCKDB_VERSION","duckdbApi","DUCKDB","store","pkg","sizeHint","version","installSpec","duckdbApiPromise","loadOrInstall","packageNodeModules"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QAFAC;eAAAA;;QACAC;eAAAA;;QAeGC;eAAAA;;;wBAlByD;AAElE,IAAMF,iBAAiB;AACvB,IAAMC,iBAAiB;AACvB,IAAMF,sBAAsB,AAAC,GAAoBE,OAAlBD,gBAAe,KAAkB,OAAfC;AAExD,IAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKL;IACLM,UAAU;IACVC,SAASN;IACTO,aAAaT;AACf;AAEA,IAAIU;AAIG,SAASP;IACd,IAAI,CAACO,kBAAkBA,mBAAmBC,IAAAA,uBAAa,EAAoCP,QAAQQ,IAAAA,4BAAkB;IACrH,OAAOF;AACT"}

package/dist/cjs/store/duckdb/open.js

@@ -33,6 +33,7 @@ var _nativets = require("./native.js");
 var _reconcilets = require("./reconcile.js");
 var _sqlfunctionsts = require("./sql-functions.js");
 var _storets = require("./store.js");
+var _vectorsts = require("./vectors.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -168,7 +169,7 @@ var DB_FILENAME = 'cache.duckdb';
 var SCHEMA_VERSION = '4';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
-function ensureSchema(_handle, conn, cfg) {
+function ensureSchema(handle, conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
         return _ts_generator(this, function(_state) {
@@ -203,21 +204,21 @@ function ensureSchema(_handle, conn, cfg) {
                 case 5:
                     _state.trys.push([
                         5,
-                        12,
                         13,
-                        14
+                        14,
+                        15
                     ]);
                     _iterator = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
                     _state.label = 6;
                 case 6:
                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
                         3,
-                        11
+                        12
                     ];
                     feature = _step.value;
                     if (!(feature.name === 'embed')) return [
                         3,
-                        8
+                        9
                     ];
                     return [
                         4,
@@ -226,37 +227,43 @@ function ensureSchema(_handle, conn, cfg) {
                 case 7:
                     _state.sent();
                     return [
-                        3,
-                        10
+                        4,
+                        (0, _vectorsts.createVectorWriteStage)(handle.duckdb, _typests.STORE_DIMS)
                     ];
                 case 8:
+                    _state.sent();
+                    return [
+                        3,
+                        11
+                    ];
+                case 9:
                     return [
                         4,
                         feature.schema(conn)
                     ];
-                case 9:
-                    _state.sent();
-                    _state.label = 10;
                 case 10:
+                    _state.sent();
+                    _state.label = 11;
+                case 11:
                     _iteratorNormalCompletion = true;
                     return [
                         3,
                         6
                     ];
-                case 11:
+                case 12:
                     return [
                         3,
-                        14
+                        15
                     ];
-                case 12:
+                case 13:
                     err = _state.sent();
                     _didIteratorError = true;
                     _iteratorError = err;
                     return [
                         3,
-                        14
+                        15
                     ];
-                case 13:
+                case 14:
                     try {
                         if (!_iteratorNormalCompletion && _iterator.return != null) {
                             _iterator.return();
@@ -269,41 +276,41 @@ function ensureSchema(_handle, conn, cfg) {
                     return [
                         7
                     ];
-                case 14:
+                case 15:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'schema_version')
                     ];
-                case 15:
+                case 16:
                     if (!(_state.sent() === null)) return [
                         3,
-                        17
+                        18
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'schema_version', SCHEMA_VERSION)
                     ];
-                case 16:
-                    _state.sent();
-                    _state.label = 17;
                 case 17:
+                    _state.sent();
+                    _state.label = 18;
+                case 18:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'features')
                     ];
-                case 18:
+                case 19:
                     if (!(_state.sent() === null)) return [
                         3,
-                        20
+                        21
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES))
                     ];
-                case 19:
-                    _state.sent();
-                    _state.label = 20;
                 case 20:
+                    _state.sent();
+                    _state.label = 21;
+                case 21:
                     return [
                         2
                     ];

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","createVectorWriteStage","duckdb","schema","getMeta","setMeta","featureSignature","FEATURES","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAiBaA;eAAAA;;QAGAC;eAAAA;;QAyEAC;eAAAA;;QAcSC;eAAAA;;;uBAzGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;yBACW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEhC,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;;YAKxE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;wBAAMC,IAAAA,iCAAsB,EAACR,OAAOS,MAAM,EAAEF,mBAAU;;;oBAAtD;oBACA;;;;;oBAEF;;wBAAMJ,QAAQO,MAAM,CAACT;;;oBAArB;;;oBARG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAUA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMf,MAAoB;;;YACvCA,OAAOS,MAAM,CAACO,cAAc;YAC5BhB,OAAOiB,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLd,QACAQ,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BV,SAAS;;;;;;oBACFc;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACrB;;;oBAAxB;oBACMR,OAAO8B,IAAAA,8BAAgB,EAACtB;oBAC9B;;wBAAO;4BAAET,QAAQ;gCAAEiB,UAAAA;gCAAUR,QAAAA;4BAAO;4BAAGR,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMR,MAAM;4BAAEE,UAAAA;4BAAUR,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMc;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAJ,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFqB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAActC,QAAQC;eAASqC,IAAAA,oBAAW,EAACtC,OAAOiB,QAAQ,EAAEjB,OAAOS,MAAM,EAAER;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}

package/dist/cjs/store/duckdb/vectors.d.cts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/cjs/store/duckdb/vectors.d.ts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/cjs/store/duckdb/vectors.js

@@ -9,6 +9,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get createVectorWriteStage () {
+        return createVectorWriteStage;
+    },
     get scanCandidates () {
         return scanCandidates;
     },
@@ -21,7 +24,6 @@ _export(exports, {
 });
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
-var _batchts = require("./batch.js");
 var _nativets = require("./native.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -182,6 +184,25 @@ function _unsupported_iterable_to_array(o, minLen) {
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 var untyped = undefined;
+var VECTOR_WRITE_STAGE = '_sense_vector_write_stage';
+function createVectorWriteStage(duckdb, dims) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        duckdb.run("CREATE TEMP TABLE ".concat(VECTOR_WRITE_STAGE, ' ("path" TEXT, chunk INTEGER, vector FLOAT[').concat(dims, "])"))
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 // The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);
 // zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.
 function padded(values, dims) {
@@ -206,7 +227,7 @@ function dequantize(row, dims) {
 }
 function writeVectorBatch(duckdb, conn, dims, rows) {
     return _async_to_generator(function() {
-        var _ref, arrayValue, ARRAY, DOUBLE, rewritten;
+        var _ref, ARRAY, FLOAT, vectorType;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -218,71 +239,62 @@ function writeVectorBatch(duckdb, conn, dims, rows) {
                         (0, _nativets.duckdbApi)()
                     ];
                 case 1:
-                    _ref = _state.sent(), arrayValue = _ref.arrayValue, ARRAY = _ref.ARRAY, DOUBLE = _ref.DOUBLE;
-                    rewritten = (0, _batchts.rewriteUpdate)('UPDATE embeddings SET vector = ? WHERE "path" = ? AND chunk = ?', rows.length);
-                    if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');
+                    _ref = _state.sent(), ARRAY = _ref.ARRAY, FLOAT = _ref.FLOAT;
+                    vectorType = ARRAY(FLOAT, dims);
                     return [
                         4,
                         (0, _transactionts.withTransaction)(conn, function() {
                             return _async_to_generator(function() {
-                                var stmt, values, types, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row;
+                                var appender, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row;
                                 return _ts_generator(this, function(_state) {
                                     switch(_state.label){
                                         case 0:
                                             return [
                                                 4,
-                                                duckdb.prepare(rewritten.sql)
+                                                duckdb.run("DELETE FROM ".concat(VECTOR_WRITE_STAGE))
                                             ];
                                         case 1:
-                                            stmt = _state.sent();
-                                            _state.label = 2;
-                                        case 2:
-                                            _state.trys.push([
-                                                2,
-                                                ,
+                                            _state.sent();
+                                            return [
                                                 4,
-                                                5
-                                            ]);
-                                            values = [];
-                                            types = [];
-                                            _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                                duckdb.createAppender(VECTOR_WRITE_STAGE)
+                                            ];
+                                        case 2:
+                                            appender = _state.sent();
                                             try {
-                                                for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                                                    row = _step.value;
-                                                    values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);
-                                                    types.push(ARRAY(DOUBLE, dims), untyped, untyped);
-                                                }
-                                            } catch (err) {
-                                                _didIteratorError = true;
-                                                _iteratorError = err;
-                                            } finally{
+                                                _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                                                 try {
-                                                    if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                                        _iterator.return();
+                                                    for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                                        row = _step.value;
+                                                        appender.appendVarchar(row.path);
+                                                        appender.appendInteger(row.chunk);
+                                                        appender.appendArray(dequantize(row, dims), vectorType);
+                                                        appender.endRow();
                                                     }
+                                                } catch (err) {
+                                                    _didIteratorError = true;
+                                                    _iteratorError = err;
                                                 } finally{
-                                                    if (_didIteratorError) {
-                                                        throw _iteratorError;
+                                                    try {
+                                                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                                            _iterator.return();
+                                                        }
+                                                    } finally{
+                                                        if (_didIteratorError) {
+                                                            throw _iteratorError;
+                                                        }
                                                     }
                                                 }
+                                                appender.flushSync();
+                                            } finally{
+                                                appender.closeSync();
                                             }
-                                            stmt.bind(values, types);
                                             return [
                                                 4,
-                                                stmt.run()
+                                                duckdb.run("UPDATE embeddings SET vector = staged.vector FROM ".concat(VECTOR_WRITE_STAGE, ' AS staged WHERE embeddings."path" = staged."path" AND embeddings.chunk = staged.chunk'))
                                             ];
                                         case 3:
                                             _state.sent();
-                                            return [
-                                                3,
-                                                5
-                                            ];
-                                        case 4:
-                                            stmt.destroySync();
-                                            return [
-                                                7
-                                            ];
-                                        case 5:
                                             return [
                                                 2
                                             ];

package/dist/cjs/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","duckdbApi","rewriteUpdate","Error","withTransaction","stmt","types","prepare","sql","push","path","chunk","bind","run","destroySync","qv","fetch","allowed","allowedList","query","queryNorm","score","reader","size","sqrt","reduce","sum","value","map","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkEsBA;eAAAA;;QAmCAC;eAAAA;;QA5DAC;eAAAA;;;6BAxCU;yBAEO;uBACT;wBACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,IAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,IAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCtB;AAEJ;AAIO,SAAeL,iBAAiB4B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEyB,IAAsB;;YAE/E,MAA9BC,YAAYC,OAAOC,QACrBC;;;;oBAFN,IAAIJ,KAAKlB,MAAM,KAAK,GAAG;;;oBACe;;wBAAMuB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBC,YAAYE,IAAAA,sBAAa,EAAC,mEAAmEN,KAAKlB,MAAM;oBAC9G,IAAI,CAACsB,WAAW,MAAM,IAAIG,MAAM;oBAEhC;;wBAAMC,IAAAA,8BAAe,EAACT,MAAM;;oCACpBU,MAEEnC,QACAoC,OACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB;;;;4CAJA;;gDAAMS,OAAOa,OAAO,CAACP,UAAUQ,GAAG;;;4CAAzCH,OAAO;;;;;;;;;4CAELnC;4CACAoC;4CACD,kCAAA,2BAAA;;gDAAL,IAAK,YAAaV,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;oDAAbX,MAAN;oDACHf,OAAOuC,IAAI,CAACZ,WAAWb,WAAWC,KAAKd,QAAQc,IAAIyB,IAAI,EAAEzB,IAAI0B,KAAK;oDAClEL,MAAMG,IAAI,CAACX,MAAMC,QAAQ5B,OAAOJ,SAASA;gDAC3C;;gDAHK;gDAAA;;;yDAAA,6BAAA;wDAAA;;;wDAAA;8DAAA;;;;4CAILsC,KAAKO,IAAI,CAAC1C,QAAQoC;4CAClB;;gDAAMD,KAAKQ,GAAG;;;4CAAd;;;;;;4CAEAR,KAAKS,WAAW;;;;;;;;;;4BAEpB;;;;oBAdA;;;;;;IAeF;;AAIO,SAAelD,eAAe8B,MAAwB,EAAEqB,EAAgB,EAAE5C,IAAY,EAAE6C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BpB,YAAYC,OAAOC,QACrBmB,aACAC,OACAC,WACAC,OACAb,KAaAH,MAEEnC,QACAoC,OAEAgB,QACA1B;;;;oBAzBR,IAAIqB,WAAWA,QAAQM,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMtB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBmB,cAAcD,UAAW,qBAAGA;oBAC5BE,QAAQlD,OAAO8C,IAAI5C;oBACnBiD,YAAY5C,KAAKgD,IAAI,CAACL,MAAMM,MAAM,CAAC,SAACC,KAAKC;+BAAUD,MAAMC,QAAQA;uBAAO;oBACxEN,QAAQD,cAAc,IAAI,QAAQ;oBAClCZ,MAAM,AAAC,4QAOoBS,OAFmBI,OAAM,yEAEsC,OAA/DJ,UAAU,AAAC,OAA6C,OAAvCtC,SAAS,UAAUuC,YAAYxC,MAAM,KAAM,IAAG;oBAMnF;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB,AAAC,qBAAIkD,cAAc;wBAAUvB,WAAWsB;8BAAU,qBAAGD;wBAAaF;;oBAC1FV,QAAsB,AAAC,qBAAIc,cAAc;wBAAUtB,MAAMC,QAAQ5B;8BAAS,qBAAG+C,YAAYU,GAAG,CAAC;+BAAM7D;;wBAAUA;;oBACnHsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKwB,aAAa;;;oBAAjCP,SAAS;oBACT1B,OAAO0B,OAAOQ,eAAe;oBACnC;;wBAAOlC,KAAKgC,GAAG,CAAC,SAACG;mCAAO;gCAAErB,MAAMqB,EAAErB,IAAI;gCAAEsB,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEV,KAAK;4BAAE;;;;oBAE/GhB,KAAKS,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAejD,YAAY6B,MAAwB,EAAEC,IAAgB,EAAExB,IAAY,EAAEuC,IAAY,EAAE2B,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9B3C,YAAYC,OAAOC,QACrB0C,aACAvB,aACAV,KAWAH,MAEEnC,QACAoC,OAEAgB,QACA1B;;;;oBA1BR,IAAIyC,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACM,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAM5B,KAAKY,OAAO,CAAC;;;oBAAhC+B,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAChC;;;oBAAnC6B,aAAc;oBACpB,IAAIA,WAAW7D,MAAM,KAAK,GAAG;;;;oBACvB8D,UAAUG,IAAAA,uBAAY,EAACJ,YAAYX,GAAG,CAAC,SAAC3C;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMa,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrB0C,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9B1B,cAAcmB,KAAKpB,OAAO,GAAI,qBAAGoB,KAAKpB,OAAO;oBAC7CT,MAAM,AAAC,+BAIPiC,OAJqCD,QAAQZ,GAAG,CAAC;+BAAM;uBAAO7C,IAAI,CAAC,OAAM,kTAKzEsD,OADAI,YAAY/D,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAY8D,YAAY/D,MAAM,KAAM,IAAG,YACd,OAAtE2D,KAAKpB,OAAO,GAAG,AAAC,OAA+C,OAAzCtC,SAAS,YAAYuC,YAAYxC,MAAM,KAAM,IAAG;oBAM/D;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELnC,SAAwB,AAAC,qBAAGsE,QAAQZ,GAAG,CAAC,SAACiB;+BAAMhD,WAAW5B,OAAO4E,GAAG1E;;wBAASuC;uBAAM,qBAAG+B,cAAa,qBAAGvB;wBAAamB,KAAKS,CAAC;;oBACzHxC,QAAsB,AAAC,qBAAGkC,QAAQZ,GAAG,CAAC;+BAAM9B,MAAMC,QAAQ5B;;wBAAQJ;uBAAS,qBAAG0E,YAAYb,GAAG,CAAC;+BAAM7D;yBAAU,qBAAGmD,YAAYU,GAAG,CAAC;+BAAM7D;;wBAAUA;;oBACvJsC,KAAKO,IAAI,CAAC1C,QAAQoC;oBACH;;wBAAMD,KAAKwB,aAAa;;;oBAAjCP,SAAS;oBACT1B,OAAO0B,OAAOQ,eAAe;oBACnC;;wBAAOlC,KAAKgC,GAAG,CAAC,SAACG;mCAAO;gCAAErB,MAAMqB,EAAErB,IAAI;gCAAEyB,YAAYC,IAAAA,mBAAQ,EAACL,EAAEV,KAAK;4BAAE;;;;oBAEtEhB,KAAKS,WAAW;;;;;;;;;;IAEpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\nconst VECTOR_WRITE_STAGE = '_sense_vector_write_stage';\n\n// Created once per native connection outside write transactions; its lifetime follows that connection.\nexport async function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void> {\n  await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} (\"path\" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// The native appender needs an explicit FLOAT array type: inference from an integer first component\n// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { ARRAY, FLOAT } = await duckdbApi();\n  const vectorType = ARRAY(FLOAT, dims);\n\n  await withTransaction(conn, async () => {\n    await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);\n    const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);\n    try {\n      for (const row of rows) {\n        appender.appendVarchar(row.path);\n        appender.appendInteger(row.chunk);\n        appender.appendArray(dequantize(row, dims), vectorType);\n        appender.endRow();\n      }\n      appender.flushSync();\n    } finally {\n      appender.closeSync();\n    }\n    await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings.\"path\" = staged.\"path\" AND embeddings.chunk = staged.chunk`);\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["createVectorWriteStage","scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","VECTOR_WRITE_STAGE","duckdb","dims","run","padded","values","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","conn","rows","ARRAY","FLOAT","vectorType","duckdbApi","withTransaction","appender","createAppender","appendVarchar","path","appendInteger","chunk","appendArray","endRow","flushSync","closeSync","qv","fetch","allowed","arrayValue","DOUBLE","allowedList","query","queryNorm","score","sql","stmt","types","reader","size","sqrt","reduce","sum","value","prepare","map","bind","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","destroySync","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkBsBA;eAAAA;;QAqDAC;eAAAA;;QAmCAC;eAAAA;;QA5DAC;eAAAA;;;6BA7CU;yBAEO;wBACb;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAChB,IAAMC,qBAAqB;AAGpB,SAAeN,uBAAuBO,MAAwB,EAAEC,IAAY;;;;;oBACjF;;wBAAMD,OAAOE,GAAG,CAAC,AAAC,qBAAoFD,OAAhEF,oBAAmB,+CAAkD,OAALE,MAAK;;;oBAA3G;;;;;;IACF;;AAEA,wHAAwH;AACxH,oIAAoI;AACpI,SAASE,OAAOC,MAAyB,EAAEH,IAAY;IACrD,IAAMI,MAAM,IAAIC,MAAcL,MAAMM,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEV,OAAOO,IAAKH,GAAG,CAACG,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,IAAMkB,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOrB,OACLG,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCzB;AAEJ;AAIO,SAAeL,iBAAiBI,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAE2B,IAAsB;;YAE5F,MAAjBC,OAAOC,OACTC;;;;oBAFN,IAAIH,KAAKjB,MAAM,KAAK,GAAG;;;oBACE;;wBAAMqB,IAAAA,mBAAS;;;oBAAf,OAAA,eAAjBH,QAAiB,KAAjBA,OAAOC,QAAU,KAAVA;oBACTC,aAAaF,MAAMC,OAAO7B;oBAEhC;;wBAAMgC,IAAAA,8BAAe,EAACN,MAAM;;oCAEpBO,UAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMhB;;;;4CAHb;;gDAAMlB,OAAOE,GAAG,CAAC,AAAC,eAAiC,OAAnBH;;;4CAAhC;4CACiB;;gDAAMC,OAAOmC,cAAc,CAACpC;;;4CAAvCmC,WAAW;4CACjB,IAAI;gDACG,kCAAA,2BAAA;;oDAAL,IAAK,YAAaN,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;wDAAbV,MAAN;wDACHgB,SAASE,aAAa,CAAClB,IAAImB,IAAI;wDAC/BH,SAASI,aAAa,CAACpB,IAAIqB,KAAK;wDAChCL,SAASM,WAAW,CAACvB,WAAWC,KAAKjB,OAAO8B;wDAC5CG,SAASO,MAAM;oDACjB;;oDALK;oDAAA;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;gDAMLP,SAASQ,SAAS;4CACpB,SAAU;gDACRR,SAASS,SAAS;4CACpB;4CACA;;gDAAM3C,OAAOE,GAAG,CAAC,AAAC,qDAAuE,OAAnBH,oBAAmB;;;4CAAzF;;;;;;4BACF;;;;oBAfA;;;;;;IAgBF;;AAIO,SAAeL,eAAeM,MAAwB,EAAE4C,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BC,YAAYlB,OAAOmB,QACrBC,aACAC,OACAC,WACAC,OACAC,KAaAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBAzBR,IAAIkB,WAAWA,QAAQW,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMzB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrBC,cAAcH,UAAW,qBAAGA;oBAC5BI,QAAQ/C,OAAOyC,IAAI3C;oBACnBkD,YAAY1C,KAAKiD,IAAI,CAACR,MAAMS,MAAM,CAAC,SAACC,KAAKC;+BAAUD,MAAMC,QAAQA;uBAAO;oBACxET,QAAQD,cAAc,IAAI,QAAQ;oBAClCE,MAAM,AAAC,4QAOoBP,OAFmBM,OAAM,yEAEsC,OAA/DN,UAAU,AAAC,OAA6C,OAAvClC,SAAS,UAAUqC,YAAYtC,MAAM,KAAM,IAAG;oBAMnF;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAI+C,cAAc;wBAAUJ,WAAWG;8BAAU,qBAAGD;wBAAaJ;;oBAC1FU,QAAsB,AAAC,qBAAIJ,cAAc;wBAAUtB,MAAMmB,QAAQ/C;8BAAS,qBAAGgD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACnHyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAE+B,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAE/GE,KAAKmB,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAe9E,YAAYK,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAEoC,IAAY,EAAEqC,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9B9B,YAAYlB,OAAOmB,QACrB8B,aACA7B,aACAI,KAWAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBA1BR,IAAI8C,KAAK5B,OAAO,IAAI4B,KAAK5B,OAAO,CAACW,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAM9B,KAAKmC,OAAO,CAAC;;;oBAAhCa,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAC1C;;;oBAAnCuC,aAAc;oBACpB,IAAIA,WAAWjE,MAAM,KAAK,GAAG;;;;oBACvBkE,UAAUG,IAAAA,uBAAY,EAACJ,YAAYb,GAAG,CAAC,SAAC7C;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMW,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrB8B,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9BhC,cAAcyB,KAAK5B,OAAO,GAAI,qBAAG4B,KAAK5B,OAAO;oBAC7CO,MAAM,AAAC,+BAIPyB,OAJqCD,QAAQd,GAAG,CAAC;+BAAM;uBAAO/C,IAAI,CAAC,OAAM,kTAKzE0D,OADAI,YAAYnE,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAYkE,YAAYnE,MAAM,KAAM,IAAG,YACd,OAAtE+D,KAAK5B,OAAO,GAAG,AAAC,OAA+C,OAAzClC,SAAS,YAAYqC,YAAYtC,MAAM,KAAM,IAAG;oBAM/D;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAGyE,QAAQd,GAAG,CAAC,SAACmB;+BAAMnC,WAAW5C,OAAO+E,GAAGjF;;wBAASoC;uBAAM,qBAAGyC,cAAa,qBAAG7B;wBAAayB,KAAKS,CAAC;;oBACzH5B,QAAsB,AAAC,qBAAGsB,QAAQd,GAAG,CAAC;+BAAMlC,MAAMmB,QAAQ/C;;wBAAQJ;uBAAS,qBAAGiF,YAAYf,GAAG,CAAC;+BAAMlE;yBAAU,qBAAGoD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACvJyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAEkC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAEtEE,KAAKmB,WAAW;;;;;;;;;;IAEpB"}

package/dist/cjs/watch.js

@@ -9,10 +9,10 @@ Object.defineProperty(exports, "runWatch", {
     }
 });
 var _nodefs = require("node:fs");
-var _indexts = require("./config/index.js");
 var _errorsts = require("./errors.js");
 var _guardedtickts = require("./lib/guarded-tick.js");
-var _indexts1 = require("./store/index.js");
+var _watchnotificationts = require("./lib/watch-notification.js");
+var _indexts = require("./store/index.js");
 var _watchclaimts = require("./watch-claim.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
@@ -188,7 +188,7 @@ function runWatch(_0) {
                     ]);
                     return [
                         4,
-                        (0, _indexts1.openStore)(cfg)
+                        (0, _indexts.openStore)(cfg)
                     ];
                 case 3:
                     _ref = _state.sent(), initialStore = _ref.store, dbPath = _ref.dbPath, initialWarnings = _ref.warnings, initialParsed = _ref.parsed;
@@ -207,7 +207,7 @@ function runWatch(_0) {
                     ];
                     return [
                         4,
-                        (0, _indexts1.docCount)(initialStore)
+                        (0, _indexts.docCount)(initialStore)
                     ];
                 case 5:
                     _tmp = _state.sent();
@@ -256,7 +256,7 @@ function runWatch(_0) {
                                         ]);
                                         return [
                                             4,
-                                            (0, _indexts1.openStore)(cfg)
+                                            (0, _indexts.openStore)(cfg)
                                         ];
                                     case 1:
                                         _ref = _state.sent(), store = _ref.store, parsed = _ref.parsed, warnings = _ref.warnings;
@@ -278,7 +278,7 @@ function runWatch(_0) {
                                         };
                                         return [
                                             4,
-                                            (0, _indexts1.docCount)(store)
+                                            (0, _indexts.docCount)(store)
                                         ];
                                     case 3:
                                         onEvent.apply(void 0, [
@@ -354,7 +354,7 @@ function runWatch(_0) {
                     watcher = (0, _nodefs.watch)(rootDir, {
                         recursive: true
                     }, function(_event, filename) {
-                        if (typeof filename === 'string' && filename.startsWith(_indexts.STATE_DIR)) return;
+                        if (!(0, _watchnotificationts.shouldReconcileWatchNotification)(filename)) return;
                         scheduleReconcile();
                     });
                     reconcileTimer = setInterval((0, _guardedtickts.guardedTick)(function() {
@@ -410,7 +410,7 @@ function runWatch(_0) {
                                                     ];
                                                     return [
                                                         4,
-                                                        (0, _indexts1.openStore)(cfg)
+                                                        (0, _indexts.openStore)(cfg)
                                                     ];
                                                 case 3:
                                                     _ref = _state.sent(), finalStore = _ref.store;

package/dist/cjs/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, openStore } from './store/index.ts';\nimport { startWatchClaim, WATCH_HEARTBEAT_INTERVAL_MS } from './watch-claim.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\n\nexport type WatchEvent = { type: 'started'; rootDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. DuckDB and Turso hand the cache file to\n// one process at a time, so every reconcile cycle opens and closes instead of holding an idle lock.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const reconcileIntervalMs = opts.heartbeatIntervalMs ?? WATCH_HEARTBEAT_INTERVAL_MS;\n  if (!Number.isFinite(reconcileIntervalMs) || reconcileIntervalMs <= 0) throw new SenseError('CONFIG_INVALID', 'watch heartbeatIntervalMs must be a finite positive number');\n  // Reconciliation may be parked for a caller, but live ownership must renew before fixed expiry.\n  const claimIntervalMs = Math.min(reconcileIntervalMs, WATCH_HEARTBEAT_INTERVAL_MS);\n  const rootDir = cfg.rootDir ?? cfg.baseDir;\n  const configDir = cfg.configDir ?? rootDir;\n  let claimFailure: Error | null = null;\n  let requestShutdown: ((reason: Error) => void) | undefined;\n  const claim = await startWatchClaim(configDir, opts.force ?? false, claimIntervalMs, (err) => {\n    claimFailure ??= err;\n    requestShutdown?.(err);\n  });\n  let claimCloseStarted = false;\n  const closeClaim = (): Promise<void> => {\n    claimCloseStarted = true;\n    return claim.close();\n  };\n  try {\n    const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n    let initialTotal = 0;\n    try {\n      // total is read before close: an event fires only once nothing is held open again.\n      initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n    } finally {\n      await initialStore.close();\n    }\n    if (claimFailure) throw claimFailure;\n\n    let stopping = false;\n    // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n    // coalesced into a single rerun instead of opening a second, overlapping connection.\n    let running: Promise<void> | null = null;\n    let queuedAlways: boolean | null = null;\n\n    const tick = async (alwaysEmit: boolean): Promise<void> => {\n      try {\n        const { store, parsed, warnings } = await openStore(cfg);\n        try {\n          if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n            onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n          }\n        } finally {\n          await store.close();\n        }\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    };\n\n    const startCycle = (alwaysEmit: boolean) => {\n      running = tick(alwaysEmit).then(() => {\n        running = null;\n        if (queuedAlways !== null && !stopping) {\n          const again = queuedAlways;\n          queuedAlways = null;\n          startCycle(again);\n        }\n      });\n    };\n\n    const requestCycle = (alwaysEmit: boolean) => {\n      if (stopping) return;\n      if (running) {\n        queuedAlways = queuedAlways || alwaysEmit;\n        return;\n      }\n      startCycle(alwaysEmit);\n    };\n\n    let debounceTimer: NodeJS.Timeout | null = null;\n    const scheduleReconcile = () => {\n      if (debounceTimer) clearTimeout(debounceTimer);\n      debounceTimer = setTimeout(() => {\n        debounceTimer = null;\n        requestCycle(true);\n      }, debounceMs);\n    };\n\n    // Ignore our own state files, or cache and claim writes would retrigger reconciliation forever.\n    // An unresolvable filename reconciles because parsing nothing costs less than missing a real edit.\n    const watcher = fsWatch(rootDir, { recursive: true }, (_event, filename) => {\n      if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n      scheduleReconcile();\n    });\n    const reconcileTimer = setInterval(\n      guardedTick(\n        async () => requestCycle(false),\n        () => stopping\n      ),\n      reconcileIntervalMs\n    );\n\n    return await new Promise<void>((resolveShutdown, rejectShutdown) => {\n      let shutdownReason: Error | null = null;\n      const shutdown = (reason?: Error) => {\n        shutdownReason ??= reason ?? null;\n        if (stopping) return;\n        stopping = true;\n        const shutdownTask = (async () => {\n          process.off('SIGINT', stopNormally);\n          process.off('SIGTERM', stopNormally);\n          opts.signal?.removeEventListener('abort', stopNormally);\n          clearInterval(reconcileTimer);\n          if (debounceTimer) clearTimeout(debounceTimer);\n          watcher.close();\n          await running;\n          let finalError: unknown;\n          try {\n            // A normal stop performs one final freshness pass. Ownership loss drains existing work\n            // but starts nothing new because the replacement watcher now owns that responsibility.\n            if (!shutdownReason) {\n              const { store: finalStore } = await openStore(cfg);\n              await finalStore.close();\n            }\n          } catch (err) {\n            finalError = err;\n          }\n          try {\n            await closeClaim();\n          } catch (err) {\n            if (finalError) throw new AggregateError([finalError, err], 'watch shutdown and claim cleanup both failed');\n            throw err;\n          }\n          if (finalError) throw finalError;\n          if (shutdownReason) throw shutdownReason;\n        })();\n        shutdownTask.then(resolveShutdown, rejectShutdown);\n      };\n      const stopNormally = () => shutdown();\n      requestShutdown = shutdown;\n      process.once('SIGINT', stopNormally);\n      process.once('SIGTERM', stopNormally);\n      if (claimFailure) shutdown(claimFailure);\n      else if (opts.signal?.aborted) shutdown();\n      else {\n        opts.signal?.addEventListener('abort', stopNormally, { once: true });\n        try {\n          onEvent({ type: 'started', rootDir, dbPath });\n          if (!stopping && (initialWarnings.length > 0 || initialParsed > 0)) {\n            onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n          }\n        } catch (err) {\n          shutdown(err instanceof Error ? err : new Error(String(err)));\n        }\n      }\n    });\n  } catch (err) {\n    if (claimCloseStarted) throw err;\n    try {\n      await closeClaim();\n    } catch (closeError) {\n      throw new AggregateError([err, closeError], 'watch setup and claim cleanup both failed');\n    }\n    throw err;\n  } finally {\n    requestShutdown = undefined;\n  }\n}\n"],"names":["runWatch","DEBOUNCE_MS","cfg","opts","onEvent","debounceMs","reconcileIntervalMs","claimIntervalMs","rootDir","configDir","claimFailure","requestShutdown","claim","claimCloseStarted","closeClaim","initialStore","dbPath","initialWarnings","initialParsed","initialTotal","stopping","running","queuedAlways","tick","startCycle","requestCycle","debounceTimer","scheduleReconcile","watcher","reconcileTimer","err","closeError","heartbeatIntervalMs","WATCH_HEARTBEAT_INTERVAL_MS","Number","isFinite","SenseError","Math","min","baseDir","startWatchClaim","force","close","openStore","store","warnings","parsed","length","docCount","alwaysEmit","type","total","message","then","again","clearTimeout","setTimeout","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","rejectShutdown","shutdownReason","shutdown","reason","shutdownTask","finalError","finalStore","process","off","stopNormally","signal","removeEventListener","clearInterval","AggregateError","once","aborted","addEventListener","Error","String","undefined"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;uBAEP;wBACC;6BACC;wBACQ;4BACyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7D,uJAAuJ;AACvJ,IAAMC,cAAc;AAeb,SAAeD;wCAASE,GAAmB;YAAEC,MAClCA,eACGA,kBACSA,2BAIZD,cACEA,gBAG6BC,aAVzCC,SACAC,YACAC,qBAGAC,iBACAC,SACAC,WACFC,cACAC,iBACEC,OAIFC,mBACEC,YAKsF,MAA3EC,cAAcC,QAAkBC,iBAAyBC,eACpEC,oBASAC,UAGAC,SACAC,cAEEC,MAeAC,YAWAC,cASFC,eACEC,mBAUAC,SAIAC,gBA8DCC,KAIEC;;;;;oBAzJuC5B,OAAAA,oEAAqB,CAAC;oBAClEC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBF;oBAChCK,uBAAsBH,4BAAAA,KAAK6B,mBAAmB,cAAxB7B,uCAAAA,4BAA4B8B,yCAA2B;oBACnF,IAAI,CAACC,OAAOC,QAAQ,CAAC7B,wBAAwBA,uBAAuB,GAAG,MAAM,IAAI8B,oBAAU,CAAC,kBAAkB;oBAC9G,gGAAgG;oBAC1F7B,kBAAkB8B,KAAKC,GAAG,CAAChC,qBAAqB2B,yCAA2B;oBAC3EzB,WAAUN,eAAAA,IAAIM,OAAO,cAAXN,0BAAAA,eAAeA,IAAIqC,OAAO;oBACpC9B,aAAYP,iBAAAA,IAAIO,SAAS,cAAbP,4BAAAA,iBAAiBM;oBAC/BE,eAA6B;oBAEnB;;wBAAM8B,IAAAA,6BAAe,EAAC/B,YAAWN,cAAAA,KAAKsC,KAAK,cAAVtC,yBAAAA,cAAc,OAAOI,iBAAiB,SAACuB;4BACpFpB,yBAAAA,0BAAAA,eAAAA,eAAiBoB;4BACjBnB,4BAAAA,sCAAAA,gBAAkBmB;wBACpB;;;oBAHMlB,QAAQ;oBAIVC,oBAAoB;oBAClBC,aAAa;wBACjBD,oBAAoB;wBACpB,OAAOD,MAAM8B,KAAK;oBACpB;;;;;;;;;oBAE4F;;wBAAMC,IAAAA,mBAAS,EAACzC;;;oBAAhB,OAAA,eAA3Ea,eAA2E,KAAlF6B,OAAqB5B,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArD4B,UAAmC3B,gBAAkB,KAA1B4B;oBAC5D3B,eAAe;;;;;;;;;yBAGFF,CAAAA,gBAAgB8B,MAAM,GAAG,KAAK7B,gBAAgB,CAAA,GAA9CD;;;;oBAAkD;;wBAAM+B,IAAAA,kBAAQ,EAACjC;;;2BAAf;;;;;;2BAA+B;;;oBADhG,mFAAmF;oBACnFI;;;;;;oBAEA;;wBAAMJ,aAAa2B,KAAK;;;oBAAxB;;;;;oBAEF,IAAIhC,cAAc,MAAMA;oBAEpBU,WAAW;oBACf,6FAA6F;oBAC7F,qFAAqF;oBACjFC,UAAgC;oBAChCC,eAA+B;oBAE7BC,OAAO,cAAO0B;;gCAEoB,MAA5BL,OAAOE,QAAQD,gBAQhBf;;;;;;;;;;wCAR6B;;4CAAMa,IAAAA,mBAAS,EAACzC;;;wCAAhB,OAAA,eAA5B0C,QAA4B,KAA5BA,OAAOE,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;;;;;;;;6CAEjBI,CAAAA,cAAcH,SAAS,KAAKD,SAASE,MAAM,GAAG,CAAA,GAA9CE;;;;;4CACQC,MAAM;4CAAcJ,QAAAA;;wCAAe;;4CAAME,IAAAA,kBAAQ,EAACJ;;;wCAA5DxC;kDAAsC+C,QAAO,oBAAuBN,WAAAA;;;;;;;;;wCAGtE;;4CAAMD,MAAMF,KAAK;;;wCAAjB;;;;;;;;;;wCAEKZ;wCACP1B,QAAQ;4CAAE8C,MAAM;4CAAmBE,SAAS,AAACtB,IAAcsB,OAAO;wCAAC;;;;;;;;;;;wBAEvE;;oBAEM5B,aAAa,qBAACyB;wBAClB5B,UAAUE,KAAK0B,YAAYI,IAAI,CAAC;4BAC9BhC,UAAU;4BACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;gCACtC,IAAMkC,QAAQhC;gCACdA,eAAe;gCACfE,WAAW8B;4BACb;wBACF;oBACF;oBAEM7B,eAAe,sBAACwB;wBACpB,IAAI7B,UAAU;wBACd,IAAIC,SAAS;4BACXC,eAAeA,gBAAgB2B;4BAC/B;wBACF;wBACAzB,WAAWyB;oBACb;oBAEIvB,gBAAuC;oBACrCC,oBAAoB;wBACxB,IAAID,eAAe6B,aAAa7B;wBAChCA,gBAAgB8B,WAAW;4BACzB9B,gBAAgB;4BAChBD,aAAa;wBACf,GAAGpB;oBACL;oBAEA,gGAAgG;oBAChG,mGAAmG;oBAC7FuB,UAAU6B,IAAAA,aAAO,EAACjD,SAAS;wBAAEkD,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpEnC;oBACF;oBACME,iBAAiBkC,YACrBC,IAAAA,0BAAW,EACT;;;;;oCAAYvC,aAAa;;;;uBACzB;+BAAML;wBAERd;oBAGK;;wBAAM,IAAI2D,QAAc,SAACC,iBAAiBC;gCAyCtChE;4BAxCT,IAAIiE,iBAA+B;4BACnC,IAAMC,WAAW,kBAACC;gCAChBF,2BAAAA,4BAAAA,iBAAAA,iBAAmBE,mBAAAA,oBAAAA,SAAU;gCAC7B,IAAIlD,UAAU;gCACdA,WAAW;gCACX,IAAMmD,eAAe,AAAC;;4CAGpBpE,cAKIqE,YAK8B,MAAfC,YAGV3C,KAKAA;;;;oDApBT4C,QAAQC,GAAG,CAAC,UAAUC;oDACtBF,QAAQC,GAAG,CAAC,WAAWC;qDACvBzE,eAAAA,KAAK0E,MAAM,cAAX1E,mCAAAA,aAAa2E,mBAAmB,CAAC,SAASF;oDAC1CG,cAAclD;oDACd,IAAIH,eAAe6B,aAAa7B;oDAChCE,QAAQc,KAAK;oDACb;;wDAAMrB;;;oDAAN;;;;;;;;;yDAKM,CAAC+C,gBAAD;;;;oDAC4B;;wDAAMzB,IAAAA,mBAAS,EAACzC;;;oDAAhB,OAAA,eAAfuE,aAAe,KAAtB7B;oDACR;;wDAAM6B,WAAW/B,KAAK;;;oDAAtB;;;;;;;;oDAEKZ;oDACP0C,aAAa1C;;;;;;;;;;;;oDAGb;;wDAAMhB;;;oDAAN;;;;;;oDACOgB;oDACP,IAAI0C,YAAY,MAAM,IAAIQ;wDAAgBR;wDAAY1C;uDAAM;oDAC5D,MAAMA;;oDAER,IAAI0C,YAAY,MAAMA;oDACtB,IAAIJ,gBAAgB,MAAMA;;;;;;oCAC5B;;gCACAG,aAAalB,IAAI,CAACa,iBAAiBC;4BACrC;4BACA,IAAMS,eAAe;uCAAMP;;4BAC3B1D,kBAAkB0D;4BAClBK,QAAQO,IAAI,CAAC,UAAUL;4BACvBF,QAAQO,IAAI,CAAC,WAAWL;4BACxB,IAAIlE,cAAc2D,SAAS3D;iCACtB,KAAIP,eAAAA,KAAK0E,MAAM,cAAX1E,mCAAAA,aAAa+E,OAAO,EAAEb;iCAC1B;oCACHlE;iCAAAA,gBAAAA,KAAK0E,MAAM,cAAX1E,oCAAAA,cAAagF,gBAAgB,CAAC,SAASP,cAAc;oCAAEK,MAAM;gCAAK;gCAClE,IAAI;oCACF7E,QAAQ;wCAAE8C,MAAM;wCAAW1C,SAAAA;wCAASQ,QAAAA;oCAAO;oCAC3C,IAAI,CAACI,YAAaH,CAAAA,gBAAgB8B,MAAM,GAAG,KAAK7B,gBAAgB,CAAA,GAAI;wCAClEd,QAAQ;4CAAE8C,MAAM;4CAAcJ,QAAQ5B;4CAAeiC,OAAOhC;4CAAc0B,UAAU5B;wCAAgB;oCACtG;gCACF,EAAE,OAAOa,KAAK;oCACZuC,SAASvC,AAAG,YAAHA,KAAesD,SAAQtD,MAAM,IAAIsD,MAAMC,OAAOvD;gCACzD;4BACF;wBACF;;;oBArDA;;wBAAO;;;oBAsDAA;oBACP,IAAIjB,mBAAmB,MAAMiB;;;;;;;;;oBAE3B;;wBAAMhB;;;oBAAN;;;;;;oBACOiB;oBACP,MAAM,IAAIiD;wBAAgBlD;wBAAKC;uBAAa;;oBAE9C,MAAMD;;oBAENnB,kBAAkB2E;;;;;;;;;;IAEtB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { shouldReconcileWatchNotification } from './lib/watch-notification.ts';\nimport { docCount, openStore } from './store/index.ts';\nimport { startWatchClaim, WATCH_HEARTBEAT_INTERVAL_MS } from './watch-claim.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\n\nexport type WatchEvent = { type: 'started'; rootDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. DuckDB and Turso hand the cache file to\n// one process at a time, so every reconcile cycle opens and closes instead of holding an idle lock.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const reconcileIntervalMs = opts.heartbeatIntervalMs ?? WATCH_HEARTBEAT_INTERVAL_MS;\n  if (!Number.isFinite(reconcileIntervalMs) || reconcileIntervalMs <= 0) throw new SenseError('CONFIG_INVALID', 'watch heartbeatIntervalMs must be a finite positive number');\n  // Reconciliation may be parked for a caller, but live ownership must renew before fixed expiry.\n  const claimIntervalMs = Math.min(reconcileIntervalMs, WATCH_HEARTBEAT_INTERVAL_MS);\n  const rootDir = cfg.rootDir ?? cfg.baseDir;\n  const configDir = cfg.configDir ?? rootDir;\n  let claimFailure: Error | null = null;\n  let requestShutdown: ((reason: Error) => void) | undefined;\n  const claim = await startWatchClaim(configDir, opts.force ?? false, claimIntervalMs, (err) => {\n    claimFailure ??= err;\n    requestShutdown?.(err);\n  });\n  let claimCloseStarted = false;\n  const closeClaim = (): Promise<void> => {\n    claimCloseStarted = true;\n    return claim.close();\n  };\n  try {\n    const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n    let initialTotal = 0;\n    try {\n      // total is read before close: an event fires only once nothing is held open again.\n      initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n    } finally {\n      await initialStore.close();\n    }\n    if (claimFailure) throw claimFailure;\n\n    let stopping = false;\n    // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n    // coalesced into a single rerun instead of opening a second, overlapping connection.\n    let running: Promise<void> | null = null;\n    let queuedAlways: boolean | null = null;\n\n    const tick = async (alwaysEmit: boolean): Promise<void> => {\n      try {\n        const { store, parsed, warnings } = await openStore(cfg);\n        try {\n          if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n            onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n          }\n        } finally {\n          await store.close();\n        }\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    };\n\n    const startCycle = (alwaysEmit: boolean) => {\n      running = tick(alwaysEmit).then(() => {\n        running = null;\n        if (queuedAlways !== null && !stopping) {\n          const again = queuedAlways;\n          queuedAlways = null;\n          startCycle(again);\n        }\n      });\n    };\n\n    const requestCycle = (alwaysEmit: boolean) => {\n      if (stopping) return;\n      if (running) {\n        queuedAlways = queuedAlways || alwaysEmit;\n        return;\n      }\n      startCycle(alwaysEmit);\n    };\n\n    let debounceTimer: NodeJS.Timeout | null = null;\n    const scheduleReconcile = () => {\n      if (debounceTimer) clearTimeout(debounceTimer);\n      debounceTimer = setTimeout(() => {\n        debounceTimer = null;\n        requestCycle(true);\n      }, debounceMs);\n    };\n\n    // Ignore our own state files, or cache and claim writes would retrigger reconciliation forever.\n    // An unresolvable filename reconciles because parsing nothing costs less than missing a real edit.\n    const watcher = fsWatch(rootDir, { recursive: true }, (_event, filename) => {\n      if (!shouldReconcileWatchNotification(filename)) return;\n      scheduleReconcile();\n    });\n    const reconcileTimer = setInterval(\n      guardedTick(\n        async () => requestCycle(false),\n        () => stopping\n      ),\n      reconcileIntervalMs\n    );\n\n    return await new Promise<void>((resolveShutdown, rejectShutdown) => {\n      let shutdownReason: Error | null = null;\n      const shutdown = (reason?: Error) => {\n        shutdownReason ??= reason ?? null;\n        if (stopping) return;\n        stopping = true;\n        const shutdownTask = (async () => {\n          process.off('SIGINT', stopNormally);\n          process.off('SIGTERM', stopNormally);\n          opts.signal?.removeEventListener('abort', stopNormally);\n          clearInterval(reconcileTimer);\n          if (debounceTimer) clearTimeout(debounceTimer);\n          watcher.close();\n          await running;\n          let finalError: unknown;\n          try {\n            // A normal stop performs one final freshness pass. Ownership loss drains existing work\n            // but starts nothing new because the replacement watcher now owns that responsibility.\n            if (!shutdownReason) {\n              const { store: finalStore } = await openStore(cfg);\n              await finalStore.close();\n            }\n          } catch (err) {\n            finalError = err;\n          }\n          try {\n            await closeClaim();\n          } catch (err) {\n            if (finalError) throw new AggregateError([finalError, err], 'watch shutdown and claim cleanup both failed');\n            throw err;\n          }\n          if (finalError) throw finalError;\n          if (shutdownReason) throw shutdownReason;\n        })();\n        shutdownTask.then(resolveShutdown, rejectShutdown);\n      };\n      const stopNormally = () => shutdown();\n      requestShutdown = shutdown;\n      process.once('SIGINT', stopNormally);\n      process.once('SIGTERM', stopNormally);\n      if (claimFailure) shutdown(claimFailure);\n      else if (opts.signal?.aborted) shutdown();\n      else {\n        opts.signal?.addEventListener('abort', stopNormally, { once: true });\n        try {\n          onEvent({ type: 'started', rootDir, dbPath });\n          if (!stopping && (initialWarnings.length > 0 || initialParsed > 0)) {\n            onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n          }\n        } catch (err) {\n          shutdown(err instanceof Error ? err : new Error(String(err)));\n        }\n      }\n    });\n  } catch (err) {\n    if (claimCloseStarted) throw err;\n    try {\n      await closeClaim();\n    } catch (closeError) {\n      throw new AggregateError([err, closeError], 'watch setup and claim cleanup both failed');\n    }\n    throw err;\n  } finally {\n    requestShutdown = undefined;\n  }\n}\n"],"names":["runWatch","DEBOUNCE_MS","cfg","opts","onEvent","debounceMs","reconcileIntervalMs","claimIntervalMs","rootDir","configDir","claimFailure","requestShutdown","claim","claimCloseStarted","closeClaim","initialStore","dbPath","initialWarnings","initialParsed","initialTotal","stopping","running","queuedAlways","tick","startCycle","requestCycle","debounceTimer","scheduleReconcile","watcher","reconcileTimer","err","closeError","heartbeatIntervalMs","WATCH_HEARTBEAT_INTERVAL_MS","Number","isFinite","SenseError","Math","min","baseDir","startWatchClaim","force","close","openStore","store","warnings","parsed","length","docCount","alwaysEmit","type","total","message","then","again","clearTimeout","setTimeout","fsWatch","recursive","_event","filename","shouldReconcileWatchNotification","setInterval","guardedTick","Promise","resolveShutdown","rejectShutdown","shutdownReason","shutdown","reason","shutdownTask","finalError","finalStore","process","off","stopNormally","signal","removeEventListener","clearInterval","AggregateError","once","aborted","addEventListener","Error","String","undefined"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;wBAEN;6BACC;mCACqB;uBACb;4BACyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7D,uJAAuJ;AACvJ,IAAMC,cAAc;AAeb,SAAeD;wCAASE,GAAmB;YAAEC,MAClCA,eACGA,kBACSA,2BAIZD,cACEA,gBAG6BC,aAVzCC,SACAC,YACAC,qBAGAC,iBACAC,SACAC,WACFC,cACAC,iBACEC,OAIFC,mBACEC,YAKsF,MAA3EC,cAAcC,QAAkBC,iBAAyBC,eACpEC,oBASAC,UAGAC,SACAC,cAEEC,MAeAC,YAWAC,cASFC,eACEC,mBAUAC,SAIAC,gBA8DCC,KAIEC;;;;;oBAzJuC5B,OAAAA,oEAAqB,CAAC;oBAClEC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBF;oBAChCK,uBAAsBH,4BAAAA,KAAK6B,mBAAmB,cAAxB7B,uCAAAA,4BAA4B8B,yCAA2B;oBACnF,IAAI,CAACC,OAAOC,QAAQ,CAAC7B,wBAAwBA,uBAAuB,GAAG,MAAM,IAAI8B,oBAAU,CAAC,kBAAkB;oBAC9G,gGAAgG;oBAC1F7B,kBAAkB8B,KAAKC,GAAG,CAAChC,qBAAqB2B,yCAA2B;oBAC3EzB,WAAUN,eAAAA,IAAIM,OAAO,cAAXN,0BAAAA,eAAeA,IAAIqC,OAAO;oBACpC9B,aAAYP,iBAAAA,IAAIO,SAAS,cAAbP,4BAAAA,iBAAiBM;oBAC/BE,eAA6B;oBAEnB;;wBAAM8B,IAAAA,6BAAe,EAAC/B,YAAWN,cAAAA,KAAKsC,KAAK,cAAVtC,yBAAAA,cAAc,OAAOI,iBAAiB,SAACuB;4BACpFpB,yBAAAA,0BAAAA,eAAAA,eAAiBoB;4BACjBnB,4BAAAA,sCAAAA,gBAAkBmB;wBACpB;;;oBAHMlB,QAAQ;oBAIVC,oBAAoB;oBAClBC,aAAa;wBACjBD,oBAAoB;wBACpB,OAAOD,MAAM8B,KAAK;oBACpB;;;;;;;;;oBAE4F;;wBAAMC,IAAAA,kBAAS,EAACzC;;;oBAAhB,OAAA,eAA3Ea,eAA2E,KAAlF6B,OAAqB5B,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArD4B,UAAmC3B,gBAAkB,KAA1B4B;oBAC5D3B,eAAe;;;;;;;;;yBAGFF,CAAAA,gBAAgB8B,MAAM,GAAG,KAAK7B,gBAAgB,CAAA,GAA9CD;;;;oBAAkD;;wBAAM+B,IAAAA,iBAAQ,EAACjC;;;2BAAf;;;;;;2BAA+B;;;oBADhG,mFAAmF;oBACnFI;;;;;;oBAEA;;wBAAMJ,aAAa2B,KAAK;;;oBAAxB;;;;;oBAEF,IAAIhC,cAAc,MAAMA;oBAEpBU,WAAW;oBACf,6FAA6F;oBAC7F,qFAAqF;oBACjFC,UAAgC;oBAChCC,eAA+B;oBAE7BC,OAAO,cAAO0B;;gCAEoB,MAA5BL,OAAOE,QAAQD,gBAQhBf;;;;;;;;;;wCAR6B;;4CAAMa,IAAAA,kBAAS,EAACzC;;;wCAAhB,OAAA,eAA5B0C,QAA4B,KAA5BA,OAAOE,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;;;;;;;;6CAEjBI,CAAAA,cAAcH,SAAS,KAAKD,SAASE,MAAM,GAAG,CAAA,GAA9CE;;;;;4CACQC,MAAM;4CAAcJ,QAAAA;;wCAAe;;4CAAME,IAAAA,iBAAQ,EAACJ;;;wCAA5DxC;kDAAsC+C,QAAO,oBAAuBN,WAAAA;;;;;;;;;wCAGtE;;4CAAMD,MAAMF,KAAK;;;wCAAjB;;;;;;;;;;wCAEKZ;wCACP1B,QAAQ;4CAAE8C,MAAM;4CAAmBE,SAAS,AAACtB,IAAcsB,OAAO;wCAAC;;;;;;;;;;;wBAEvE;;oBAEM5B,aAAa,qBAACyB;wBAClB5B,UAAUE,KAAK0B,YAAYI,IAAI,CAAC;4BAC9BhC,UAAU;4BACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;gCACtC,IAAMkC,QAAQhC;gCACdA,eAAe;gCACfE,WAAW8B;4BACb;wBACF;oBACF;oBAEM7B,eAAe,sBAACwB;wBACpB,IAAI7B,UAAU;wBACd,IAAIC,SAAS;4BACXC,eAAeA,gBAAgB2B;4BAC/B;wBACF;wBACAzB,WAAWyB;oBACb;oBAEIvB,gBAAuC;oBACrCC,oBAAoB;wBACxB,IAAID,eAAe6B,aAAa7B;wBAChCA,gBAAgB8B,WAAW;4BACzB9B,gBAAgB;4BAChBD,aAAa;wBACf,GAAGpB;oBACL;oBAEA,gGAAgG;oBAChG,mGAAmG;oBAC7FuB,UAAU6B,IAAAA,aAAO,EAACjD,SAAS;wBAAEkD,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,CAACC,IAAAA,qDAAgC,EAACD,WAAW;wBACjDjC;oBACF;oBACME,iBAAiBiC,YACrBC,IAAAA,0BAAW,EACT;;;;;oCAAYtC,aAAa;;;;uBACzB;+BAAML;wBAERd;oBAGK;;wBAAM,IAAI0D,QAAc,SAACC,iBAAiBC;gCAyCtC/D;4BAxCT,IAAIgE,iBAA+B;4BACnC,IAAMC,WAAW,kBAACC;gCAChBF,2BAAAA,4BAAAA,iBAAAA,iBAAmBE,mBAAAA,oBAAAA,SAAU;gCAC7B,IAAIjD,UAAU;gCACdA,WAAW;gCACX,IAAMkD,eAAe,AAAC;;4CAGpBnE,cAKIoE,YAK8B,MAAfC,YAGV1C,KAKAA;;;;oDApBT2C,QAAQC,GAAG,CAAC,UAAUC;oDACtBF,QAAQC,GAAG,CAAC,WAAWC;qDACvBxE,eAAAA,KAAKyE,MAAM,cAAXzE,mCAAAA,aAAa0E,mBAAmB,CAAC,SAASF;oDAC1CG,cAAcjD;oDACd,IAAIH,eAAe6B,aAAa7B;oDAChCE,QAAQc,KAAK;oDACb;;wDAAMrB;;;oDAAN;;;;;;;;;yDAKM,CAAC8C,gBAAD;;;;oDAC4B;;wDAAMxB,IAAAA,kBAAS,EAACzC;;;oDAAhB,OAAA,eAAfsE,aAAe,KAAtB5B;oDACR;;wDAAM4B,WAAW9B,KAAK;;;oDAAtB;;;;;;;;oDAEKZ;oDACPyC,aAAazC;;;;;;;;;;;;oDAGb;;wDAAMhB;;;oDAAN;;;;;;oDACOgB;oDACP,IAAIyC,YAAY,MAAM,IAAIQ;wDAAgBR;wDAAYzC;uDAAM;oDAC5D,MAAMA;;oDAER,IAAIyC,YAAY,MAAMA;oDACtB,IAAIJ,gBAAgB,MAAMA;;;;;;oCAC5B;;gCACAG,aAAajB,IAAI,CAACY,iBAAiBC;4BACrC;4BACA,IAAMS,eAAe;uCAAMP;;4BAC3BzD,kBAAkByD;4BAClBK,QAAQO,IAAI,CAAC,UAAUL;4BACvBF,QAAQO,IAAI,CAAC,WAAWL;4BACxB,IAAIjE,cAAc0D,SAAS1D;iCACtB,KAAIP,eAAAA,KAAKyE,MAAM,cAAXzE,mCAAAA,aAAa8E,OAAO,EAAEb;iCAC1B;oCACHjE;iCAAAA,gBAAAA,KAAKyE,MAAM,cAAXzE,oCAAAA,cAAa+E,gBAAgB,CAAC,SAASP,cAAc;oCAAEK,MAAM;gCAAK;gCAClE,IAAI;oCACF5E,QAAQ;wCAAE8C,MAAM;wCAAW1C,SAAAA;wCAASQ,QAAAA;oCAAO;oCAC3C,IAAI,CAACI,YAAaH,CAAAA,gBAAgB8B,MAAM,GAAG,KAAK7B,gBAAgB,CAAA,GAAI;wCAClEd,QAAQ;4CAAE8C,MAAM;4CAAcJ,QAAQ5B;4CAAeiC,OAAOhC;4CAAc0B,UAAU5B;wCAAgB;oCACtG;gCACF,EAAE,OAAOa,KAAK;oCACZsC,SAAStC,AAAG,YAAHA,KAAeqD,SAAQrD,MAAM,IAAIqD,MAAMC,OAAOtD;gCACzD;4BACF;wBACF;;;oBArDA;;wBAAO;;;oBAsDAA;oBACP,IAAIjB,mBAAmB,MAAMiB;;;;;;;;;oBAE3B;;wBAAMhB;;;oBAAN;;;;;;oBACOiB;oBACP,MAAM,IAAIgD;wBAAgBjD;wBAAKC;uBAAa;;oBAE9C,MAAMD;;oBAENnB,kBAAkB0E;;;;;;;;;;IAEtB"}

package/dist/esm/lib/watch-notification.d.ts

@@ -0,0 +1 @@
+export declare function shouldReconcileWatchNotification(filename: string | Buffer | null): boolean;

package/dist/esm/lib/watch-notification.js

@@ -0,0 +1,4 @@
+import { STATE_DIR } from '../config/types.js';
+export function shouldReconcileWatchNotification(filename) {
+    return typeof filename !== 'string' || !filename.startsWith(STATE_DIR);
+}

package/dist/esm/lib/watch-notification.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/lib/watch-notification.ts"],"sourcesContent":["import { STATE_DIR } from '../config/types.ts';\n\nexport function shouldReconcileWatchNotification(filename: string | Buffer | null): boolean {\n  return typeof filename !== 'string' || !filename.startsWith(STATE_DIR);\n}\n"],"names":["STATE_DIR","shouldReconcileWatchNotification","filename","startsWith"],"mappings":"AAAA,SAASA,SAAS,QAAQ,qBAAqB;AAE/C,OAAO,SAASC,iCAAiCC,QAAgC;IAC/E,OAAO,OAAOA,aAAa,YAAY,CAACA,SAASC,UAAU,CAACH;AAC9D"}

package/dist/esm/store/duckdb/native.d.ts

@@ -1,4 +1,4 @@
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
-export declare const DUCKDB_VERSION = "1.5.5-r.4";
-export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.4";
+export declare const DUCKDB_VERSION = "1.5.5-r.5";
+export declare const DUCKDB_INSTALL_SPEC = "@duckdb/node-api@1.5.5-r.5";
 export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/esm/store/duckdb/native.js

@@ -1,6 +1,6 @@
 import { loadOrInstall, packageNodeModules } from '../native.js';
 export const DUCKDB_PACKAGE = '@duckdb/node-api';
-export const DUCKDB_VERSION = '1.5.5-r.4';
+export const DUCKDB_VERSION = '1.5.5-r.5';
 export const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;
 const DUCKDB = {
     store: 'duckdb',

package/dist/esm/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_VERSION = '1.5.5-r.4';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  version: DUCKDB_VERSION,\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB_VERSION","DUCKDB_INSTALL_SPEC","DUCKDB","store","pkg","sizeHint","version","installSpec","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AACjD,OAAO,MAAMC,iBAAiB,YAAY;AAC1C,OAAO,MAAMC,sBAAsB,GAAGF,eAAe,CAAC,EAAEC,gBAAgB,CAAC;AAEzE,MAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKL;IACLM,UAAU;IACVC,SAASN;IACTO,aAAaN;AACf;AAEA,IAAIO;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBX,cAAiDK,QAAQJ;IACnG,OAAOU;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_VERSION = '1.5.5-r.5';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@${DUCKDB_VERSION}`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  version: DUCKDB_VERSION,\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB_VERSION","DUCKDB_INSTALL_SPEC","DUCKDB","store","pkg","sizeHint","version","installSpec","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AACjD,OAAO,MAAMC,iBAAiB,YAAY;AAC1C,OAAO,MAAMC,sBAAsB,GAAGF,eAAe,CAAC,EAAEC,gBAAgB,CAAC;AAEzE,MAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKL;IACLM,UAAU;IACVC,SAASN;IACTO,aAAaN;AACf;AAEA,IAAIO;AAEJ,mIAAmI;AACnI,wKAAwK;AACxK,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBX,cAAiDK,QAAQJ;IACnG,OAAOU;AACT"}

package/dist/esm/store/duckdb/open.js

@@ -9,13 +9,14 @@ import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
 import { duckdbDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
+import { createVectorWriteStage } from './vectors.js';
 export const DB_FILENAME = 'cache.duckdb';
 // Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
 // The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
 export const SCHEMA_VERSION = '4';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
-async function ensureSchema(_handle, conn, cfg) {
+async function ensureSchema(handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
@@ -25,6 +26,7 @@ async function ensureSchema(_handle, conn, cfg) {
         // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.
         if (feature.name === 'embed') {
             await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY ("path", chunk))`);
+            await createVectorWriteStage(handle.duckdb, STORE_DIMS);
             continue;
         }
         await feature.schema(conn);

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","createVectorWriteStage","DB_FILENAME","SCHEMA_VERSION","ensureSchema","handle","conn","cfg","exec","feature","name","duckdb","schema","close","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AACzC,SAASC,sBAAsB,QAAQ,eAAe;AAEtD,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,MAAoB,EAAEC,IAAgB,EAAEC,GAAW;IAC7E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWnB,eAAeiB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEpB,WAAW,+BAA+B,CAAC;YAChM,MAAMa,uBAAuBI,OAAOM,MAAM,EAAEvB;YAC5C;QACF;QACA,MAAMqB,QAAQG,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,iFAAiF;AACjF,eAAesB,MAAMR,MAAoB;IACvCA,OAAOM,MAAM,CAACG,cAAc;IAC5BT,OAAOU,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIX;IACJ,IAAII;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFb,SAAS,MAAMI,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBY;QACxB,MAAML,OAAOX,iBAAiBgB;QAC9B,OAAO;YAAEN,QAAQ;gBAAEU;gBAAUJ;YAAO;YAAGL;QAAK;IAC9C,EAAE,OAAOe,KAAK;QACZ,MAAMR,MAAM;YAAEE;YAAUJ;QAAO;QAC/B,MAAMU;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUxB;IACVyB,eAAexB;IACfyB,kBAAkB9B;IAClBmB;IACAJ;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFgB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FlB;IACAJ,aAAa,CAACK,QAAQC,OAASN,YAAYK,OAAOU,QAAQ,EAAEV,OAAOM,MAAM,EAAEL;AAC7E,EAAE;AAEF,OAAO,eAAeyB,WAAWxB,GAAmB;IAClD,OAAOf,gBAAgBe,KAAKkB;AAC9B"}

package/dist/esm/store/duckdb/vectors.d.ts

@@ -1,5 +1,6 @@
 import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void>;
 export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
 export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {

package/dist/esm/store/duckdb/vectors.js

@@ -1,6 +1,5 @@
 import { withTransaction } from '../transaction.js';
 import { asCosine, sampleEvenly } from '../vectors.js';
-import { rewriteUpdate } from './batch.js';
 import { duckdbApi } from './native.js';
 // This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and
 // scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.
@@ -9,6 +8,11 @@ import { duckdbApi } from './native.js';
 // A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
 // the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
 const untyped = undefined;
+const VECTOR_WRITE_STAGE = '_sense_vector_write_stage';
+// Created once per native connection outside write transactions; its lifetime follows that connection.
+export async function createVectorWriteStage(duckdb, dims) {
+    await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} ("path" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);
+}
 // The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);
 // zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.
 function padded(values, dims) {
@@ -27,27 +31,27 @@ function dequantize(row, dims) {
     const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
     return padded(Array.from(q, (v)=>v * row.scale), dims);
 }
-// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the
-// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
+// The native appender needs an explicit FLOAT array type: inference from an integer first component
+// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.
 export async function writeVectorBatch(duckdb, conn, dims, rows) {
     if (rows.length === 0) return;
-    const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
-    const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE "path" = ? AND chunk = ?', rows.length);
-    if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');
+    const { ARRAY, FLOAT } = await duckdbApi();
+    const vectorType = ARRAY(FLOAT, dims);
     await withTransaction(conn, async ()=>{
-        const stmt = await duckdb.prepare(rewritten.sql);
+        await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);
+        const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);
         try {
-            const values = [];
-            const types = [];
             for (const row of rows){
-                values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);
-                types.push(ARRAY(DOUBLE, dims), untyped, untyped);
+                appender.appendVarchar(row.path);
+                appender.appendInteger(row.chunk);
+                appender.appendArray(dequantize(row, dims), vectorType);
+                appender.endRow();
             }
-            stmt.bind(values, types);
-            await stmt.run();
+            appender.flushSync();
         } finally{
-            stmt.destroySync();
+            appender.closeSync();
         }
+        await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings."path" = staged."path" AND embeddings.chunk = staged.chunk`);
     });
 }
 // Best chunk per file by cosine, with the earliest chunk winning exact score ties.

package/dist/esm/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's rewriteUpdate), bound directly against the\n// native connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type -- auto inference misreads an all-integer first component as INTEGER and silently truncates the rest (verified live: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","rewriteUpdate","duckdbApi","untyped","undefined","padded","values","dims","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","query","queryNorm","sqrt","reduce","sum","value","score","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,wHAAwH;AACxH,oIAAoI;AACpI,SAASC,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACP,OAAOQ,MAAM,EAAEP,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEd,IAAY;IACnD,MAAMe,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOtB,OACLI,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCtB;AAEJ;AAEA,oIAAoI;AACpI,8QAA8Q;AAC9Q,OAAO,eAAeuB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAE0B,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMmC,YAAYpC,cAAc,mEAAmEgC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAMxC,gBAAgBkC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMnC,SAAwB,EAAE;YAChC,MAAMoC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtB3B,OAAOqC,IAAI,CAACT,WAAWd,WAAWC,KAAKd,QAAQc,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ7B,OAAOJ,SAASA;YAC3C;YACAoC,KAAKO,IAAI,CAACxC,QAAQoC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,mFAAmF;AACnF,oGAAoG;AACpG,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAMoD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMG,QAAQlD,OAAO6C,IAAI3C;IACzB,MAAMiD,YAAY5C,KAAK6C,IAAI,CAACF,MAAMG,MAAM,CAAC,CAACC,KAAKC,QAAUD,MAAMC,QAAQA,OAAO;IAC9E,MAAMC,QAAQL,cAAc,IAAI,QAAQ;IACxC,MAAMf,MAAM,CAAC;;;;;oDAKqC,EAAEoB,MAAM;;iCAE3B,EAAET,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKvF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAKkD,cAAc,IAAI,EAAE,GAAG;gBAACtB,WAAWqB;aAAO;eAAMD;YAAaH;SAAM;QACtG,MAAMT,QAAsB;eAAKc,cAAc,IAAI,EAAE,GAAG;gBAACrB,MAAMC,QAAQ7B;aAAM;eAAM+C,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC3HoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAEuB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACjH,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeuB,YAAYxC,MAAwB,EAAEC,IAAgB,EAAEzB,IAAY,EAAEqC,IAAY,EAAE4B,IAAgE;IACxK,IAAIA,KAAKpB,OAAO,IAAIoB,KAAKpB,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMoB,aAAa,MAAMzC,KAAKQ,OAAO,CAAC;IACtC,MAAMkC,aAAc,MAAMD,WAAWE,GAAG,CAAC/B;IACzC,IAAI8B,WAAW5D,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAM8D,UAAU5E,aAAa0E,YAAYZ,GAAG,CAAC,CAACzC,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMlC;IAC5C,MAAM2E,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAMxB,cAAckB,KAAKpB,OAAO,GAAG;WAAIoB,KAAKpB,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAEmC,QAAQd,GAAG,CAAC,IAAM,OAAO3C,IAAI,CAAC,MAAM;;;;MAI3E,EAAE0D,YAAY/D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAY8D,YAAY/D,MAAM,GAAG,GAAG,GAAG;MACtF,EAAE0D,KAAKpB,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;;WAKnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMnC,SAAwB;eAAIsE,QAAQd,GAAG,CAAC,CAACiB,IAAM7C,WAAW7B,OAAO0E,GAAGxE;YAASqC;eAASiC;eAAgBvB;YAAakB,KAAKQ,CAAC;SAAC;QAChI,MAAMtC,QAAsB;eAAIkC,QAAQd,GAAG,CAAC,IAAM3B,MAAMC,QAAQ7B;YAAQJ;eAAY0E,YAAYf,GAAG,CAAC,IAAM3D;eAAamD,YAAYQ,GAAG,CAAC,IAAM3D;YAAUA;SAAQ;QAC/JoC,KAAKO,IAAI,CAACxC,QAAQoC;QAClB,MAAMqB,SAAS,MAAMxB,KAAKyB,aAAa;QACvC,MAAM/B,OAAO8B,OAAOE,eAAe;QACnC,OAAOhC,KAAK6B,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEtB,MAAMsB,EAAEtB,IAAI;gBAAE0B,YAAYvE,SAASmE,EAAEL,KAAK;YAAE,CAAA;IACxE,SAAU;QACRtB,KAAKS,WAAW;IAClB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\nconst VECTOR_WRITE_STAGE = '_sense_vector_write_stage';\n\n// Created once per native connection outside write transactions; its lifetime follows that connection.\nexport async function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void> {\n  await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} (\"path\" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// The native appender needs an explicit FLOAT array type: inference from an integer first component\n// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { ARRAY, FLOAT } = await duckdbApi();\n  const vectorType = ARRAY(FLOAT, dims);\n\n  await withTransaction(conn, async () => {\n    await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);\n    const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);\n    try {\n      for (const row of rows) {\n        appender.appendVarchar(row.path);\n        appender.appendInteger(row.chunk);\n        appender.appendArray(dequantize(row, dims), vectorType);\n        appender.endRow();\n      }\n      appender.flushSync();\n    } finally {\n      appender.closeSync();\n    }\n    await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings.\"path\" = staged.\"path\" AND embeddings.chunk = staged.chunk`);\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","asCosine","sampleEvenly","duckdbApi","untyped","undefined","VECTOR_WRITE_STAGE","createVectorWriteStage","duckdb","dims","run","padded","values","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","conn","rows","ARRAY","FLOAT","vectorType","appender","createAppender","appendVarchar","path","appendInteger","chunk","appendArray","endRow","flushSync","closeSync","scanCandidates","qv","fetch","allowed","size","arrayValue","DOUBLE","allowedList","query","queryNorm","sqrt","reduce","sum","value","score","sql","stmt","prepare","types","map","bind","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","destroySync","scanSimilar","opts","targetStmt","targetRows","all","targets","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AACvD,SAASC,SAAS,QAAQ,cAAc;AAExC,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAChB,MAAMC,qBAAqB;AAE3B,uGAAuG;AACvG,OAAO,eAAeC,uBAAuBC,MAAwB,EAAEC,IAAY;IACjF,MAAMD,OAAOE,GAAG,CAAC,CAAC,kBAAkB,EAAEJ,mBAAmB,2CAA2C,EAAEG,KAAK,EAAE,CAAC;AAChH;AAEA,wHAAwH;AACxH,oIAAoI;AACpI,SAASE,OAAOC,MAAyB,EAAEH,IAAY;IACrD,MAAMI,MAAM,IAAIC,MAAcL,MAAMM,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEV,OAAOO,IAAKH,GAAG,CAACG,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,MAAMkB,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOrB,OACLG,MAAMS,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCzB;AAEJ;AAEA,oGAAoG;AACpG,gHAAgH;AAChH,OAAO,eAAe0B,iBAAiB3B,MAAwB,EAAE4B,IAAgB,EAAE3B,IAAY,EAAE4B,IAAsB;IACrH,IAAIA,KAAKlB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEmB,KAAK,EAAEC,KAAK,EAAE,GAAG,MAAMpC;IAC/B,MAAMqC,aAAaF,MAAMC,OAAO9B;IAEhC,MAAMT,gBAAgBoC,MAAM;QAC1B,MAAM5B,OAAOE,GAAG,CAAC,CAAC,YAAY,EAAEJ,oBAAoB;QACpD,MAAMmC,WAAW,MAAMjC,OAAOkC,cAAc,CAACpC;QAC7C,IAAI;YACF,KAAK,MAAMoB,OAAOW,KAAM;gBACtBI,SAASE,aAAa,CAACjB,IAAIkB,IAAI;gBAC/BH,SAASI,aAAa,CAACnB,IAAIoB,KAAK;gBAChCL,SAASM,WAAW,CAACtB,WAAWC,KAAKjB,OAAO+B;gBAC5CC,SAASO,MAAM;YACjB;YACAP,SAASQ,SAAS;QACpB,SAAU;YACRR,SAASS,SAAS;QACpB;QACA,MAAM1C,OAAOE,GAAG,CAAC,CAAC,kDAAkD,EAAEJ,mBAAmB,sFAAsF,CAAC;IAClL;AACF;AAEA,mFAAmF;AACnF,oGAAoG;AACpG,OAAO,eAAe6C,eAAe3C,MAAwB,EAAE4C,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEC,UAAU,EAAElB,KAAK,EAAEmB,MAAM,EAAE,GAAG,MAAMtD;IAC5C,MAAMuD,cAAcJ,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMK,QAAQhD,OAAOyC,IAAI3C;IACzB,MAAMmD,YAAY3C,KAAK4C,IAAI,CAACF,MAAMG,MAAM,CAAC,CAACC,KAAKC,QAAUD,MAAMC,QAAQA,OAAO;IAC9E,MAAMC,QAAQL,cAAc,IAAI,QAAQ;IACxC,MAAMM,MAAM,CAAC;;;;;oDAKqC,EAAED,MAAM;;iCAE3B,EAAEX,UAAU,CAAC,IAAI,EAAElC,SAAS,UAAUsC,YAAYvC,MAAM,GAAG,GAAG,GAAG;;;;;WAKvF,CAAC;IACV,MAAMgD,OAAO,MAAM3D,OAAO4D,OAAO,CAACF;IAClC,IAAI;QACF,MAAMtD,SAAwB;eAAKgD,cAAc,IAAI,EAAE,GAAG;gBAACJ,WAAWG;aAAO;eAAMD;YAAaL;SAAM;QACtG,MAAMgB,QAAsB;eAAKT,cAAc,IAAI,EAAE,GAAG;gBAACtB,MAAMmB,QAAQhD;aAAM;eAAMiD,YAAYY,GAAG,CAAC,IAAMlE;YAAUA;SAAQ;QAC3H+D,KAAKI,IAAI,CAAC3D,QAAQyD;QAClB,MAAMG,SAAS,MAAML,KAAKM,aAAa;QACvC,MAAMpC,OAAOmC,OAAOE,eAAe;QACnC,OAAOrC,KAAKiC,GAAG,CAAC,CAACK,IAAO,CAAA;gBAAE/B,MAAM+B,EAAE/B,IAAI;gBAAEgC,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAY9E,SAAS0E,EAAEV,KAAK;YAAE,CAAA;IACjH,SAAU;QACRE,KAAKa,WAAW;IAClB;AACF;AAEA,mJAAmJ;AACnJ,oMAAoM;AACpM,OAAO,eAAeC,YAAYzE,MAAwB,EAAE4B,IAAgB,EAAE3B,IAAY,EAAEmC,IAAY,EAAEsC,IAAgE;IACxK,IAAIA,KAAK5B,OAAO,IAAI4B,KAAK5B,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAM4B,aAAa,MAAM/C,KAAKgC,OAAO,CAAC;IACtC,MAAMgB,aAAc,MAAMD,WAAWE,GAAG,CAACzC;IACzC,IAAIwC,WAAWjE,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMmE,UAAUpF,aAAakF,YAAYd,GAAG,CAAC,CAAC5C,MAAQA,IAAIG,MAAM;IAEhE,MAAM,EAAE2B,UAAU,EAAElB,KAAK,EAAEmB,MAAM,EAAE,GAAG,MAAMtD;IAC5C,MAAMoF,cAAc;WAAIL,KAAKM,OAAO;KAAC;IACrC,MAAM9B,cAAcwB,KAAK5B,OAAO,GAAG;WAAI4B,KAAK5B,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMY,MAAM,CAAC,4BAA4B,EAAEoB,QAAQhB,GAAG,CAAC,IAAM,OAAO9C,IAAI,CAAC,MAAM;;;;MAI3E,EAAE+D,YAAYpE,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAYmE,YAAYpE,MAAM,GAAG,GAAG,GAAG;MACtF,EAAE+D,KAAK5B,OAAO,GAAG,CAAC,IAAI,EAAElC,SAAS,YAAYsC,YAAYvC,MAAM,GAAG,GAAG,GAAG;;;;;WAKnE,CAAC;IACV,MAAMgD,OAAO,MAAM3D,OAAO4D,OAAO,CAACF;IAClC,IAAI;QACF,MAAMtD,SAAwB;eAAI0E,QAAQhB,GAAG,CAAC,CAACmB,IAAMjC,WAAW7C,OAAO8E,GAAGhF;YAASmC;eAAS2C;eAAgB7B;YAAawB,KAAKQ,CAAC;SAAC;QAChI,MAAMrB,QAAsB;eAAIiB,QAAQhB,GAAG,CAAC,IAAMhC,MAAMmB,QAAQhD;YAAQL;eAAYmF,YAAYjB,GAAG,CAAC,IAAMlE;eAAasD,YAAYY,GAAG,CAAC,IAAMlE;YAAUA;SAAQ;QAC/J+D,KAAKI,IAAI,CAAC3D,QAAQyD;QAClB,MAAMG,SAAS,MAAML,KAAKM,aAAa;QACvC,MAAMpC,OAAOmC,OAAOE,eAAe;QACnC,OAAOrC,KAAKiC,GAAG,CAAC,CAACK,IAAO,CAAA;gBAAE/B,MAAM+B,EAAE/B,IAAI;gBAAEmC,YAAY9E,SAAS0E,EAAEV,KAAK;YAAE,CAAA;IACxE,SAAU;QACRE,KAAKa,WAAW;IAClB;AACF"}

package/dist/esm/watch.js

@@ -1,7 +1,7 @@
 import { watch as fsWatch } from 'node:fs';
-import { STATE_DIR } from './config/index.js';
 import { SenseError } from './errors.js';
 import { guardedTick } from './lib/guarded-tick.js';
+import { shouldReconcileWatchNotification } from './lib/watch-notification.js';
 import { docCount, openStore } from './store/index.js';
 import { startWatchClaim, WATCH_HEARTBEAT_INTERVAL_MS } from './watch-claim.js';
 // Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.
@@ -97,7 +97,7 @@ export async function runWatch(cfg, opts = {}) {
         const watcher = fsWatch(rootDir, {
             recursive: true
         }, (_event, filename)=>{
-            if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;
+            if (!shouldReconcileWatchNotification(filename)) return;
             scheduleReconcile();
         });
         const reconcileTimer = setInterval(guardedTick(async ()=>requestCycle(false), ()=>stopping), reconcileIntervalMs);

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, openStore } from './store/index.ts';\nimport { startWatchClaim, WATCH_HEARTBEAT_INTERVAL_MS } from './watch-claim.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\n\nexport type WatchEvent = { type: 'started'; rootDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. DuckDB and Turso hand the cache file to\n// one process at a time, so every reconcile cycle opens and closes instead of holding an idle lock.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const reconcileIntervalMs = opts.heartbeatIntervalMs ?? WATCH_HEARTBEAT_INTERVAL_MS;\n  if (!Number.isFinite(reconcileIntervalMs) || reconcileIntervalMs <= 0) throw new SenseError('CONFIG_INVALID', 'watch heartbeatIntervalMs must be a finite positive number');\n  // Reconciliation may be parked for a caller, but live ownership must renew before fixed expiry.\n  const claimIntervalMs = Math.min(reconcileIntervalMs, WATCH_HEARTBEAT_INTERVAL_MS);\n  const rootDir = cfg.rootDir ?? cfg.baseDir;\n  const configDir = cfg.configDir ?? rootDir;\n  let claimFailure: Error | null = null;\n  let requestShutdown: ((reason: Error) => void) | undefined;\n  const claim = await startWatchClaim(configDir, opts.force ?? false, claimIntervalMs, (err) => {\n    claimFailure ??= err;\n    requestShutdown?.(err);\n  });\n  let claimCloseStarted = false;\n  const closeClaim = (): Promise<void> => {\n    claimCloseStarted = true;\n    return claim.close();\n  };\n  try {\n    const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n    let initialTotal = 0;\n    try {\n      // total is read before close: an event fires only once nothing is held open again.\n      initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n    } finally {\n      await initialStore.close();\n    }\n    if (claimFailure) throw claimFailure;\n\n    let stopping = false;\n    // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n    // coalesced into a single rerun instead of opening a second, overlapping connection.\n    let running: Promise<void> | null = null;\n    let queuedAlways: boolean | null = null;\n\n    const tick = async (alwaysEmit: boolean): Promise<void> => {\n      try {\n        const { store, parsed, warnings } = await openStore(cfg);\n        try {\n          if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n            onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n          }\n        } finally {\n          await store.close();\n        }\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    };\n\n    const startCycle = (alwaysEmit: boolean) => {\n      running = tick(alwaysEmit).then(() => {\n        running = null;\n        if (queuedAlways !== null && !stopping) {\n          const again = queuedAlways;\n          queuedAlways = null;\n          startCycle(again);\n        }\n      });\n    };\n\n    const requestCycle = (alwaysEmit: boolean) => {\n      if (stopping) return;\n      if (running) {\n        queuedAlways = queuedAlways || alwaysEmit;\n        return;\n      }\n      startCycle(alwaysEmit);\n    };\n\n    let debounceTimer: NodeJS.Timeout | null = null;\n    const scheduleReconcile = () => {\n      if (debounceTimer) clearTimeout(debounceTimer);\n      debounceTimer = setTimeout(() => {\n        debounceTimer = null;\n        requestCycle(true);\n      }, debounceMs);\n    };\n\n    // Ignore our own state files, or cache and claim writes would retrigger reconciliation forever.\n    // An unresolvable filename reconciles because parsing nothing costs less than missing a real edit.\n    const watcher = fsWatch(rootDir, { recursive: true }, (_event, filename) => {\n      if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n      scheduleReconcile();\n    });\n    const reconcileTimer = setInterval(\n      guardedTick(\n        async () => requestCycle(false),\n        () => stopping\n      ),\n      reconcileIntervalMs\n    );\n\n    return await new Promise<void>((resolveShutdown, rejectShutdown) => {\n      let shutdownReason: Error | null = null;\n      const shutdown = (reason?: Error) => {\n        shutdownReason ??= reason ?? null;\n        if (stopping) return;\n        stopping = true;\n        const shutdownTask = (async () => {\n          process.off('SIGINT', stopNormally);\n          process.off('SIGTERM', stopNormally);\n          opts.signal?.removeEventListener('abort', stopNormally);\n          clearInterval(reconcileTimer);\n          if (debounceTimer) clearTimeout(debounceTimer);\n          watcher.close();\n          await running;\n          let finalError: unknown;\n          try {\n            // A normal stop performs one final freshness pass. Ownership loss drains existing work\n            // but starts nothing new because the replacement watcher now owns that responsibility.\n            if (!shutdownReason) {\n              const { store: finalStore } = await openStore(cfg);\n              await finalStore.close();\n            }\n          } catch (err) {\n            finalError = err;\n          }\n          try {\n            await closeClaim();\n          } catch (err) {\n            if (finalError) throw new AggregateError([finalError, err], 'watch shutdown and claim cleanup both failed');\n            throw err;\n          }\n          if (finalError) throw finalError;\n          if (shutdownReason) throw shutdownReason;\n        })();\n        shutdownTask.then(resolveShutdown, rejectShutdown);\n      };\n      const stopNormally = () => shutdown();\n      requestShutdown = shutdown;\n      process.once('SIGINT', stopNormally);\n      process.once('SIGTERM', stopNormally);\n      if (claimFailure) shutdown(claimFailure);\n      else if (opts.signal?.aborted) shutdown();\n      else {\n        opts.signal?.addEventListener('abort', stopNormally, { once: true });\n        try {\n          onEvent({ type: 'started', rootDir, dbPath });\n          if (!stopping && (initialWarnings.length > 0 || initialParsed > 0)) {\n            onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n          }\n        } catch (err) {\n          shutdown(err instanceof Error ? err : new Error(String(err)));\n        }\n      }\n    });\n  } catch (err) {\n    if (claimCloseStarted) throw err;\n    try {\n      await closeClaim();\n    } catch (closeError) {\n      throw new AggregateError([err, closeError], 'watch setup and claim cleanup both failed');\n    }\n    throw err;\n  } finally {\n    requestShutdown = undefined;\n  }\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","openStore","startWatchClaim","WATCH_HEARTBEAT_INTERVAL_MS","DEBOUNCE_MS","runWatch","cfg","opts","onEvent","debounceMs","reconcileIntervalMs","heartbeatIntervalMs","Number","isFinite","claimIntervalMs","Math","min","rootDir","baseDir","configDir","claimFailure","requestShutdown","claim","force","err","claimCloseStarted","closeClaim","close","store","initialStore","dbPath","warnings","initialWarnings","parsed","initialParsed","initialTotal","length","stopping","running","queuedAlways","tick","alwaysEmit","type","total","message","startCycle","then","again","requestCycle","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","watcher","recursive","_event","filename","startsWith","reconcileTimer","setInterval","Promise","resolveShutdown","rejectShutdown","shutdownReason","shutdown","reason","shutdownTask","process","off","stopNormally","signal","removeEventListener","clearInterval","finalError","finalStore","AggregateError","once","aborted","addEventListener","Error","String","closeError","undefined"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,SAAS,QAAQ,mBAAmB;AACvD,SAASC,eAAe,EAAEC,2BAA2B,QAAQ,mBAAmB;AAEhF,uJAAuJ;AACvJ,MAAMC,cAAc;AAapB,oGAAoG;AACpG,oGAAoG;AACpG,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QACzDA,eACGA,kBACSA,2BAIZD,cACEA,gBAG6BC;IAV/C,MAAMC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBH;IACtC,MAAMM,uBAAsBH,4BAAAA,KAAKI,mBAAmB,cAAxBJ,uCAAAA,4BAA4BJ;IACxD,IAAI,CAACS,OAAOC,QAAQ,CAACH,wBAAwBA,uBAAuB,GAAG,MAAM,IAAIZ,WAAW,kBAAkB;IAC9G,gGAAgG;IAChG,MAAMgB,kBAAkBC,KAAKC,GAAG,CAACN,qBAAqBP;IACtD,MAAMc,WAAUX,eAAAA,IAAIW,OAAO,cAAXX,0BAAAA,eAAeA,IAAIY,OAAO;IAC1C,MAAMC,aAAYb,iBAAAA,IAAIa,SAAS,cAAbb,4BAAAA,iBAAiBW;IACnC,IAAIG,eAA6B;IACjC,IAAIC;IACJ,MAAMC,QAAQ,MAAMpB,gBAAgBiB,YAAWZ,cAAAA,KAAKgB,KAAK,cAAVhB,yBAAAA,cAAc,OAAOO,iBAAiB,CAACU;QACpFJ,yBAAAA,0BAAAA,eAAAA,eAAiBI;QACjBH,4BAAAA,sCAAAA,gBAAkBG;IACpB;IACA,IAAIC,oBAAoB;IACxB,MAAMC,aAAa;QACjBD,oBAAoB;QACpB,OAAOH,MAAMK,KAAK;IACpB;IACA,IAAI;QACF,MAAM,EAAEC,OAAOC,YAAY,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjC,UAAUK;QAC1G,IAAI6B,eAAe;QACnB,IAAI;YACF,mFAAmF;YACnFA,eAAeH,gBAAgBI,MAAM,GAAG,KAAKF,gBAAgB,IAAI,MAAMlC,SAAS6B,gBAAgB;QAClG,SAAU;YACR,MAAMA,aAAaF,KAAK;QAC1B;QACA,IAAIP,cAAc,MAAMA;QAExB,IAAIiB,WAAW;QACf,6FAA6F;QAC7F,qFAAqF;QACrF,IAAIC,UAAgC;QACpC,IAAIC,eAA+B;QAEnC,MAAMC,OAAO,OAAOC;YAClB,IAAI;gBACF,MAAM,EAAEb,KAAK,EAAEK,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAM9B,UAAUK;gBACpD,IAAI;oBACF,IAAImC,cAAcR,SAAS,KAAKF,SAASK,MAAM,GAAG,GAAG;wBACnD5B,QAAQ;4BAAEkC,MAAM;4BAAcT;4BAAQU,OAAO,MAAM3C,SAAS4B;4BAAQG;wBAAS;oBAC/E;gBACF,SAAU;oBACR,MAAMH,MAAMD,KAAK;gBACnB;YACF,EAAE,OAAOH,KAAK;gBACZhB,QAAQ;oBAAEkC,MAAM;oBAAmBE,SAAS,AAACpB,IAAcoB,OAAO;gBAAC;YACrE;QACF;QAEA,MAAMC,aAAa,CAACJ;YAClBH,UAAUE,KAAKC,YAAYK,IAAI,CAAC;gBAC9BR,UAAU;gBACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;oBACtC,MAAMU,QAAQR;oBACdA,eAAe;oBACfM,WAAWE;gBACb;YACF;QACF;QAEA,MAAMC,eAAe,CAACP;YACpB,IAAIJ,UAAU;YACd,IAAIC,SAAS;gBACXC,eAAeA,gBAAgBE;gBAC/B;YACF;YACAI,WAAWJ;QACb;QAEA,IAAIQ,gBAAuC;QAC3C,MAAMC,oBAAoB;YACxB,IAAID,eAAeE,aAAaF;YAChCA,gBAAgBG,WAAW;gBACzBH,gBAAgB;gBAChBD,aAAa;YACf,GAAGvC;QACL;QAEA,gGAAgG;QAChG,mGAAmG;QACnG,MAAM4C,UAAUzD,QAAQqB,SAAS;YAAEqC,WAAW;QAAK,GAAG,CAACC,QAAQC;YAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAAC5D,YAAY;YACpEqD;QACF;QACA,MAAMQ,iBAAiBC,YACrB5D,YACE,UAAYiD,aAAa,QACzB,IAAMX,WAER3B;QAGF,OAAO,MAAM,IAAIkD,QAAc,CAACC,iBAAiBC;gBAyCtCvD;YAxCT,IAAIwD,iBAA+B;YACnC,MAAMC,WAAW,CAACC;gBAChBF,2BAAAA,4BAAAA,iBAAAA,iBAAmBE,mBAAAA,oBAAAA,SAAU;gBAC7B,IAAI5B,UAAU;gBACdA,WAAW;gBACX,MAAM6B,eAAe,AAAC,CAAA;wBAGpB3D;oBAFA4D,QAAQC,GAAG,CAAC,UAAUC;oBACtBF,QAAQC,GAAG,CAAC,WAAWC;qBACvB9D,eAAAA,KAAK+D,MAAM,cAAX/D,mCAAAA,aAAagE,mBAAmB,CAAC,SAASF;oBAC1CG,cAAcd;oBACd,IAAIT,eAAeE,aAAaF;oBAChCI,QAAQ1B,KAAK;oBACb,MAAMW;oBACN,IAAImC;oBACJ,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,IAAI,CAACV,gBAAgB;4BACnB,MAAM,EAAEnC,OAAO8C,UAAU,EAAE,GAAG,MAAMzE,UAAUK;4BAC9C,MAAMoE,WAAW/C,KAAK;wBACxB;oBACF,EAAE,OAAOH,KAAK;wBACZiD,aAAajD;oBACf;oBACA,IAAI;wBACF,MAAME;oBACR,EAAE,OAAOF,KAAK;wBACZ,IAAIiD,YAAY,MAAM,IAAIE,eAAe;4BAACF;4BAAYjD;yBAAI,EAAE;wBAC5D,MAAMA;oBACR;oBACA,IAAIiD,YAAY,MAAMA;oBACtB,IAAIV,gBAAgB,MAAMA;gBAC5B,CAAA;gBACAG,aAAapB,IAAI,CAACe,iBAAiBC;YACrC;YACA,MAAMO,eAAe,IAAML;YAC3B3C,kBAAkB2C;YAClBG,QAAQS,IAAI,CAAC,UAAUP;YACvBF,QAAQS,IAAI,CAAC,WAAWP;YACxB,IAAIjD,cAAc4C,SAAS5C;iBACtB,KAAIb,eAAAA,KAAK+D,MAAM,cAAX/D,mCAAAA,aAAasE,OAAO,EAAEb;iBAC1B;oBACHzD;iBAAAA,gBAAAA,KAAK+D,MAAM,cAAX/D,oCAAAA,cAAauE,gBAAgB,CAAC,SAAST,cAAc;oBAAEO,MAAM;gBAAK;gBAClE,IAAI;oBACFpE,QAAQ;wBAAEkC,MAAM;wBAAWzB;wBAASa;oBAAO;oBAC3C,IAAI,CAACO,YAAaL,CAAAA,gBAAgBI,MAAM,GAAG,KAAKF,gBAAgB,CAAA,GAAI;wBAClE1B,QAAQ;4BAAEkC,MAAM;4BAAcT,QAAQC;4BAAeS,OAAOR;4BAAcJ,UAAUC;wBAAgB;oBACtG;gBACF,EAAE,OAAOR,KAAK;oBACZwC,SAASxC,eAAeuD,QAAQvD,MAAM,IAAIuD,MAAMC,OAAOxD;gBACzD;YACF;QACF;IACF,EAAE,OAAOA,KAAK;QACZ,IAAIC,mBAAmB,MAAMD;QAC7B,IAAI;YACF,MAAME;QACR,EAAE,OAAOuD,YAAY;YACnB,MAAM,IAAIN,eAAe;gBAACnD;gBAAKyD;aAAW,EAAE;QAC9C;QACA,MAAMzD;IACR,SAAU;QACRH,kBAAkB6D;IACpB;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { shouldReconcileWatchNotification } from './lib/watch-notification.ts';\nimport { docCount, openStore } from './store/index.ts';\nimport { startWatchClaim, WATCH_HEARTBEAT_INTERVAL_MS } from './watch-claim.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\n\nexport type WatchEvent = { type: 'started'; rootDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. DuckDB and Turso hand the cache file to\n// one process at a time, so every reconcile cycle opens and closes instead of holding an idle lock.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const reconcileIntervalMs = opts.heartbeatIntervalMs ?? WATCH_HEARTBEAT_INTERVAL_MS;\n  if (!Number.isFinite(reconcileIntervalMs) || reconcileIntervalMs <= 0) throw new SenseError('CONFIG_INVALID', 'watch heartbeatIntervalMs must be a finite positive number');\n  // Reconciliation may be parked for a caller, but live ownership must renew before fixed expiry.\n  const claimIntervalMs = Math.min(reconcileIntervalMs, WATCH_HEARTBEAT_INTERVAL_MS);\n  const rootDir = cfg.rootDir ?? cfg.baseDir;\n  const configDir = cfg.configDir ?? rootDir;\n  let claimFailure: Error | null = null;\n  let requestShutdown: ((reason: Error) => void) | undefined;\n  const claim = await startWatchClaim(configDir, opts.force ?? false, claimIntervalMs, (err) => {\n    claimFailure ??= err;\n    requestShutdown?.(err);\n  });\n  let claimCloseStarted = false;\n  const closeClaim = (): Promise<void> => {\n    claimCloseStarted = true;\n    return claim.close();\n  };\n  try {\n    const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n    let initialTotal = 0;\n    try {\n      // total is read before close: an event fires only once nothing is held open again.\n      initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n    } finally {\n      await initialStore.close();\n    }\n    if (claimFailure) throw claimFailure;\n\n    let stopping = false;\n    // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n    // coalesced into a single rerun instead of opening a second, overlapping connection.\n    let running: Promise<void> | null = null;\n    let queuedAlways: boolean | null = null;\n\n    const tick = async (alwaysEmit: boolean): Promise<void> => {\n      try {\n        const { store, parsed, warnings } = await openStore(cfg);\n        try {\n          if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n            onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n          }\n        } finally {\n          await store.close();\n        }\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    };\n\n    const startCycle = (alwaysEmit: boolean) => {\n      running = tick(alwaysEmit).then(() => {\n        running = null;\n        if (queuedAlways !== null && !stopping) {\n          const again = queuedAlways;\n          queuedAlways = null;\n          startCycle(again);\n        }\n      });\n    };\n\n    const requestCycle = (alwaysEmit: boolean) => {\n      if (stopping) return;\n      if (running) {\n        queuedAlways = queuedAlways || alwaysEmit;\n        return;\n      }\n      startCycle(alwaysEmit);\n    };\n\n    let debounceTimer: NodeJS.Timeout | null = null;\n    const scheduleReconcile = () => {\n      if (debounceTimer) clearTimeout(debounceTimer);\n      debounceTimer = setTimeout(() => {\n        debounceTimer = null;\n        requestCycle(true);\n      }, debounceMs);\n    };\n\n    // Ignore our own state files, or cache and claim writes would retrigger reconciliation forever.\n    // An unresolvable filename reconciles because parsing nothing costs less than missing a real edit.\n    const watcher = fsWatch(rootDir, { recursive: true }, (_event, filename) => {\n      if (!shouldReconcileWatchNotification(filename)) return;\n      scheduleReconcile();\n    });\n    const reconcileTimer = setInterval(\n      guardedTick(\n        async () => requestCycle(false),\n        () => stopping\n      ),\n      reconcileIntervalMs\n    );\n\n    return await new Promise<void>((resolveShutdown, rejectShutdown) => {\n      let shutdownReason: Error | null = null;\n      const shutdown = (reason?: Error) => {\n        shutdownReason ??= reason ?? null;\n        if (stopping) return;\n        stopping = true;\n        const shutdownTask = (async () => {\n          process.off('SIGINT', stopNormally);\n          process.off('SIGTERM', stopNormally);\n          opts.signal?.removeEventListener('abort', stopNormally);\n          clearInterval(reconcileTimer);\n          if (debounceTimer) clearTimeout(debounceTimer);\n          watcher.close();\n          await running;\n          let finalError: unknown;\n          try {\n            // A normal stop performs one final freshness pass. Ownership loss drains existing work\n            // but starts nothing new because the replacement watcher now owns that responsibility.\n            if (!shutdownReason) {\n              const { store: finalStore } = await openStore(cfg);\n              await finalStore.close();\n            }\n          } catch (err) {\n            finalError = err;\n          }\n          try {\n            await closeClaim();\n          } catch (err) {\n            if (finalError) throw new AggregateError([finalError, err], 'watch shutdown and claim cleanup both failed');\n            throw err;\n          }\n          if (finalError) throw finalError;\n          if (shutdownReason) throw shutdownReason;\n        })();\n        shutdownTask.then(resolveShutdown, rejectShutdown);\n      };\n      const stopNormally = () => shutdown();\n      requestShutdown = shutdown;\n      process.once('SIGINT', stopNormally);\n      process.once('SIGTERM', stopNormally);\n      if (claimFailure) shutdown(claimFailure);\n      else if (opts.signal?.aborted) shutdown();\n      else {\n        opts.signal?.addEventListener('abort', stopNormally, { once: true });\n        try {\n          onEvent({ type: 'started', rootDir, dbPath });\n          if (!stopping && (initialWarnings.length > 0 || initialParsed > 0)) {\n            onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n          }\n        } catch (err) {\n          shutdown(err instanceof Error ? err : new Error(String(err)));\n        }\n      }\n    });\n  } catch (err) {\n    if (claimCloseStarted) throw err;\n    try {\n      await closeClaim();\n    } catch (closeError) {\n      throw new AggregateError([err, closeError], 'watch setup and claim cleanup both failed');\n    }\n    throw err;\n  } finally {\n    requestShutdown = undefined;\n  }\n}\n"],"names":["watch","fsWatch","SenseError","guardedTick","shouldReconcileWatchNotification","docCount","openStore","startWatchClaim","WATCH_HEARTBEAT_INTERVAL_MS","DEBOUNCE_MS","runWatch","cfg","opts","onEvent","debounceMs","reconcileIntervalMs","heartbeatIntervalMs","Number","isFinite","claimIntervalMs","Math","min","rootDir","baseDir","configDir","claimFailure","requestShutdown","claim","force","err","claimCloseStarted","closeClaim","close","store","initialStore","dbPath","warnings","initialWarnings","parsed","initialParsed","initialTotal","length","stopping","running","queuedAlways","tick","alwaysEmit","type","total","message","startCycle","then","again","requestCycle","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","watcher","recursive","_event","filename","reconcileTimer","setInterval","Promise","resolveShutdown","rejectShutdown","shutdownReason","shutdown","reason","shutdownTask","process","off","stopNormally","signal","removeEventListener","clearInterval","finalError","finalStore","AggregateError","once","aborted","addEventListener","Error","String","closeError","undefined"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,gCAAgC,QAAQ,8BAA8B;AAC/E,SAASC,QAAQ,EAAEC,SAAS,QAAQ,mBAAmB;AACvD,SAASC,eAAe,EAAEC,2BAA2B,QAAQ,mBAAmB;AAEhF,uJAAuJ;AACvJ,MAAMC,cAAc;AAapB,oGAAoG;AACpG,oGAAoG;AACpG,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QACzDA,eACGA,kBACSA,2BAIZD,cACEA,gBAG6BC;IAV/C,MAAMC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBH;IACtC,MAAMM,uBAAsBH,4BAAAA,KAAKI,mBAAmB,cAAxBJ,uCAAAA,4BAA4BJ;IACxD,IAAI,CAACS,OAAOC,QAAQ,CAACH,wBAAwBA,uBAAuB,GAAG,MAAM,IAAIb,WAAW,kBAAkB;IAC9G,gGAAgG;IAChG,MAAMiB,kBAAkBC,KAAKC,GAAG,CAACN,qBAAqBP;IACtD,MAAMc,WAAUX,eAAAA,IAAIW,OAAO,cAAXX,0BAAAA,eAAeA,IAAIY,OAAO;IAC1C,MAAMC,aAAYb,iBAAAA,IAAIa,SAAS,cAAbb,4BAAAA,iBAAiBW;IACnC,IAAIG,eAA6B;IACjC,IAAIC;IACJ,MAAMC,QAAQ,MAAMpB,gBAAgBiB,YAAWZ,cAAAA,KAAKgB,KAAK,cAAVhB,yBAAAA,cAAc,OAAOO,iBAAiB,CAACU;QACpFJ,yBAAAA,0BAAAA,eAAAA,eAAiBI;QACjBH,4BAAAA,sCAAAA,gBAAkBG;IACpB;IACA,IAAIC,oBAAoB;IACxB,MAAMC,aAAa;QACjBD,oBAAoB;QACpB,OAAOH,MAAMK,KAAK;IACpB;IACA,IAAI;QACF,MAAM,EAAEC,OAAOC,YAAY,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjC,UAAUK;QAC1G,IAAI6B,eAAe;QACnB,IAAI;YACF,mFAAmF;YACnFA,eAAeH,gBAAgBI,MAAM,GAAG,KAAKF,gBAAgB,IAAI,MAAMlC,SAAS6B,gBAAgB;QAClG,SAAU;YACR,MAAMA,aAAaF,KAAK;QAC1B;QACA,IAAIP,cAAc,MAAMA;QAExB,IAAIiB,WAAW;QACf,6FAA6F;QAC7F,qFAAqF;QACrF,IAAIC,UAAgC;QACpC,IAAIC,eAA+B;QAEnC,MAAMC,OAAO,OAAOC;YAClB,IAAI;gBACF,MAAM,EAAEb,KAAK,EAAEK,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAM9B,UAAUK;gBACpD,IAAI;oBACF,IAAImC,cAAcR,SAAS,KAAKF,SAASK,MAAM,GAAG,GAAG;wBACnD5B,QAAQ;4BAAEkC,MAAM;4BAAcT;4BAAQU,OAAO,MAAM3C,SAAS4B;4BAAQG;wBAAS;oBAC/E;gBACF,SAAU;oBACR,MAAMH,MAAMD,KAAK;gBACnB;YACF,EAAE,OAAOH,KAAK;gBACZhB,QAAQ;oBAAEkC,MAAM;oBAAmBE,SAAS,AAACpB,IAAcoB,OAAO;gBAAC;YACrE;QACF;QAEA,MAAMC,aAAa,CAACJ;YAClBH,UAAUE,KAAKC,YAAYK,IAAI,CAAC;gBAC9BR,UAAU;gBACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;oBACtC,MAAMU,QAAQR;oBACdA,eAAe;oBACfM,WAAWE;gBACb;YACF;QACF;QAEA,MAAMC,eAAe,CAACP;YACpB,IAAIJ,UAAU;YACd,IAAIC,SAAS;gBACXC,eAAeA,gBAAgBE;gBAC/B;YACF;YACAI,WAAWJ;QACb;QAEA,IAAIQ,gBAAuC;QAC3C,MAAMC,oBAAoB;YACxB,IAAID,eAAeE,aAAaF;YAChCA,gBAAgBG,WAAW;gBACzBH,gBAAgB;gBAChBD,aAAa;YACf,GAAGvC;QACL;QAEA,gGAAgG;QAChG,mGAAmG;QACnG,MAAM4C,UAAUzD,QAAQqB,SAAS;YAAEqC,WAAW;QAAK,GAAG,CAACC,QAAQC;YAC7D,IAAI,CAACzD,iCAAiCyD,WAAW;YACjDN;QACF;QACA,MAAMO,iBAAiBC,YACrB5D,YACE,UAAYkD,aAAa,QACzB,IAAMX,WAER3B;QAGF,OAAO,MAAM,IAAIiD,QAAc,CAACC,iBAAiBC;gBAyCtCtD;YAxCT,IAAIuD,iBAA+B;YACnC,MAAMC,WAAW,CAACC;gBAChBF,2BAAAA,4BAAAA,iBAAAA,iBAAmBE,mBAAAA,oBAAAA,SAAU;gBAC7B,IAAI3B,UAAU;gBACdA,WAAW;gBACX,MAAM4B,eAAe,AAAC,CAAA;wBAGpB1D;oBAFA2D,QAAQC,GAAG,CAAC,UAAUC;oBACtBF,QAAQC,GAAG,CAAC,WAAWC;qBACvB7D,eAAAA,KAAK8D,MAAM,cAAX9D,mCAAAA,aAAa+D,mBAAmB,CAAC,SAASF;oBAC1CG,cAAcd;oBACd,IAAIR,eAAeE,aAAaF;oBAChCI,QAAQ1B,KAAK;oBACb,MAAMW;oBACN,IAAIkC;oBACJ,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,IAAI,CAACV,gBAAgB;4BACnB,MAAM,EAAElC,OAAO6C,UAAU,EAAE,GAAG,MAAMxE,UAAUK;4BAC9C,MAAMmE,WAAW9C,KAAK;wBACxB;oBACF,EAAE,OAAOH,KAAK;wBACZgD,aAAahD;oBACf;oBACA,IAAI;wBACF,MAAME;oBACR,EAAE,OAAOF,KAAK;wBACZ,IAAIgD,YAAY,MAAM,IAAIE,eAAe;4BAACF;4BAAYhD;yBAAI,EAAE;wBAC5D,MAAMA;oBACR;oBACA,IAAIgD,YAAY,MAAMA;oBACtB,IAAIV,gBAAgB,MAAMA;gBAC5B,CAAA;gBACAG,aAAanB,IAAI,CAACc,iBAAiBC;YACrC;YACA,MAAMO,eAAe,IAAML;YAC3B1C,kBAAkB0C;YAClBG,QAAQS,IAAI,CAAC,UAAUP;YACvBF,QAAQS,IAAI,CAAC,WAAWP;YACxB,IAAIhD,cAAc2C,SAAS3C;iBACtB,KAAIb,eAAAA,KAAK8D,MAAM,cAAX9D,mCAAAA,aAAaqE,OAAO,EAAEb;iBAC1B;oBACHxD;iBAAAA,gBAAAA,KAAK8D,MAAM,cAAX9D,oCAAAA,cAAasE,gBAAgB,CAAC,SAAST,cAAc;oBAAEO,MAAM;gBAAK;gBAClE,IAAI;oBACFnE,QAAQ;wBAAEkC,MAAM;wBAAWzB;wBAASa;oBAAO;oBAC3C,IAAI,CAACO,YAAaL,CAAAA,gBAAgBI,MAAM,GAAG,KAAKF,gBAAgB,CAAA,GAAI;wBAClE1B,QAAQ;4BAAEkC,MAAM;4BAAcT,QAAQC;4BAAeS,OAAOR;4BAAcJ,UAAUC;wBAAgB;oBACtG;gBACF,EAAE,OAAOR,KAAK;oBACZuC,SAASvC,eAAesD,QAAQtD,MAAM,IAAIsD,MAAMC,OAAOvD;gBACzD;YACF;QACF;IACF,EAAE,OAAOA,KAAK;QACZ,IAAIC,mBAAmB,MAAMD;QAC7B,IAAI;YACF,MAAME;QACR,EAAE,OAAOsD,YAAY;YACnB,MAAM,IAAIN,eAAe;gBAAClD;gBAAKwD;aAAW,EAAE;QAC9C;QACA,MAAMxD;IACR,SAAU;QACRH,kBAAkB4D;IACpB;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.24.5",
+  "version": "0.24.7",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step.",
   "keywords": [
     "markdown",
@@ -93,7 +93,7 @@
     "yaml": "^2.9.0"
   },
   "devDependencies": {
-    "@duckdb/node-api": "1.5.5-r.4",
+    "@duckdb/node-api": "1.5.5-r.5",
     "@tursodatabase/database": "*",
     "@types/markdown-it-footnote": "^3.0.4",
     "@types/mocha": "*",

package/skills/sense-setup/references/store-benchmarks.md

@@ -1,15 +1,15 @@
-<!-- sense-store-benchmark release=0.24.5 -->
+<!-- sense-store-benchmark release=0.24.7 -->
 # Current store benchmark summary
 
-The release assessment generated this file for store selection. Release `0.24.5` passed on 2026-09-11, measured on Apple M4 Pro with Node v26.7.0.
+The release assessment generated this file for store selection. Release `0.24.7` passed on 2026-09-17, measured on Apple M4 Pro with Node v26.7.0.
 
 The timing rows ran on the same 6,566-note tree. They include CLI startup and each store's complete selected path. Ranked candidates and downstream work can differ by store, so these are current operating measurements rather than an isolated database-engine contest.
 
 | Store | Cold index | Warm count | Lexical search | Semantic search | Portable semantic nDCG@10 |
 |---|---|---|---|---|---|
-| sqlite | 1,484 ms | 144 ms | 194 ms | 315 ms | 0.3306 |
-| duckdb | 2,163 ms | 162 ms | 281 ms | 386 ms | 0.3284 |
-| turso | 2,454 ms | 136 ms | 233 ms | 562 ms | 0.3307 |
+| sqlite | 1,641 ms | 156 ms | 210 ms | 349 ms | 0.3306 |
+| duckdb | 3,376 ms | 181 ms | 565 ms | 556 ms | 0.3284 |
+| turso | 3,704 ms | 154 ms | 296 ms | 664 ms | 0.3307 |
 
 Cold index is the first `status` that builds the cache. Warm count is a no-change `COUNT(*)` query. Lexical and semantic search are steady-state `sense search` commands. Lower timing is faster. Higher nDCG@10 is better; that quality column uses the same NFCorpus queries, judgments, result count, and model on every store.