sensemaking 0.21.1 → 0.22.0

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Builder } from './builder.ts';\nimport { duckdbOpenDialect, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './open.ts';\nimport { openWithBuilder } from './open.ts';\nimport { openSqlite, sqliteOpenDialect } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso, tursoOpenDialect } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n  // Same open, but keeps the builder's parse pool alive and hands it back: only a watcher, which\n  // reconciles repeatedly on the same connection, needs this.\n  openForWatch: (cfg: ResolvedConfig) => Promise<OpenResult & { builder: Builder }>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite, openForWatch: (cfg) => openWithBuilder(cfg, sqliteOpenDialect) },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb, openForWatch: (cfg) => openWithBuilder(cfg, duckdbOpenDialect) },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso, openForWatch: (cfg) => openWithBuilder(cfg, tursoOpenDialect) },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// Same capability-checked open as openStore, but for a caller (only src/watch.ts) that must\n// reconcile again later: keeps the builder's pool warm and hands it back instead of closing it.\nexport async function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & { builder: Builder }> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.openForWatch(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport type { OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","duckdbOpenDialect","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openWithBuilder","openSqlite","sqliteOpenDialect","SQLITE_CAPABILITIES","openTurso","tursoOpenDialect","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","openForWatch","cfg","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","has","openStoreForWatch","requireWatchConcurrency","clearCache","docCount","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,iBAAiB,EAAEC,UAAU,QAAQ,mBAAmB;AACjE,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,eAAe,QAAQ,YAAY;AAC5C,SAASC,UAAU,EAAEC,iBAAiB,QAAQ,mBAAmB;AACjE,SAASJ,gBAAgBK,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,kBAAkB;AAC9D,SAASP,gBAAgBQ,kBAAkB,QAAQ,mBAAmB;AAWtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcN;QAAqBO,MAAMT;QAAYU,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKV;IAAmB;IAC9HW,QAAQ;QAAEJ,cAAcV;QAAqBW,MAAMb;QAAYc,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKhB;IAAmB;IAC9HkB,OAAO;QAAEL,cAAcH;QAAoBI,MAAMN;QAAWO,cAAc,CAACC,MAAQZ,gBAAgBY,KAAKP;IAAkB;AAC5H;AAEA,SAASU,SAASC,IAAe;IAC/B,MAAMC,QAAQV,QAAQ,CAACS,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAItB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEqB,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACZ,UAAUa,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUT,GAAmB;IACjD,MAAMI,OAAOtB,UAAUkB;IACvB,MAAMK,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIvB,gBAAgBmB,QAAQ,CAACK,MAAMR,YAAY,CAACa,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMP,IAAI,CAACE;AACpB;AAEA,4FAA4F;AAC5F,gGAAgG;AAChG,OAAO,eAAeW,kBAAkBX,GAAmB;IACzD,MAAMI,OAAOtB,UAAUkB;IACvB,MAAMK,QAAQF,SAASC;IAEvB,IAAIvB,gBAAgBmB,QAAQ,CAACK,MAAMR,YAAY,CAACa,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMN,YAAY,CAACC;AAC5B;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASY,wBAAwBZ,GAAmB;IACzD,MAAMI,OAAOtB,UAAUkB;IACvB,IAAI,CAACG,SAASC,MAAMP,YAAY,CAACa,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,OAAO,EAAEqB,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AAExC,SAASC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,WAAW,EAAEC,cAAc,QAAQ,mBAAmB;AAE/D,SAASC,gBAAgB,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\nexport { clearCache } from './cache.ts';\nexport type { OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Stages } from './stages.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","clearCache","docCount","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,SAASE,UAAU,QAAQ,aAAa;AAExC,SAASC,QAAQ,QAAQ,YAAY;AACrC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,WAAW,EAAEC,cAAc,QAAQ,mBAAmB;AAG/D,SAASC,gBAAgB,QAAQ,aAAa"}

package/dist/esm/store/lock-wait.d.ts

@@ -0,0 +1,2 @@
+export declare function recordLockWaitMs(baseDir: string, ms: number): void;
+export declare function lockWaitBudgetMs(baseDir: string): number;

package/dist/esm/store/lock-wait.js

@@ -0,0 +1,36 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { STATE_DIR } from '../config/index.js';
+// duckdb/turso lock the cache file exclusively, so a waiter can't connect and read reconcile_max_ms
+// from meta; this sidecar is plain fs, readable while locked. Same derivation as busy_timeout (open.ts).
+const SIDECAR_FILENAME = 'lock-wait.json';
+const FLOOR_MS = 5000;
+const CEILING_MS = 600000;
+function sidecarPath(baseDir) {
+    return join(baseDir, STATE_DIR, SIDECAR_FILENAME);
+}
+function readRecordedMs(baseDir) {
+    try {
+        const raw = JSON.parse(readFileSync(sidecarPath(baseDir), 'utf8'));
+        return typeof raw.reconcile_max_ms === 'number' ? raw.reconcile_max_ms : 0;
+    } catch  {
+        return 0;
+    }
+}
+// Called after every reconcile's write transaction, every store alike, so a fresh duckdb/turso
+// tree starts building this the first time anything writes.
+export function recordLockWaitMs(baseDir, ms) {
+    if (ms <= readRecordedMs(baseDir)) return;
+    try {
+        writeFileSync(sidecarPath(baseDir), JSON.stringify({
+            reconcile_max_ms: ms
+        }));
+    } catch  {
+    // Best-effort: a write failure here only narrows the next waiter's budget back to the floor.
+    }
+}
+// 3x the largest reconcile this tree has recorded, floored and capped -- a waiter outlasts a
+// real reconcile and still fails behind a hung one.
+export function lockWaitBudgetMs(baseDir) {
+    return Math.min(Math.max(FLOOR_MS, 3 * readRecordedMs(baseDir)), CEILING_MS);
+}

package/dist/esm/store/lock-wait.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/lock-wait.ts"],"sourcesContent":["import { readFileSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { STATE_DIR } from '../config/index.ts';\n\n// duckdb/turso lock the cache file exclusively, so a waiter can't connect and read reconcile_max_ms\n// from meta; this sidecar is plain fs, readable while locked. Same derivation as busy_timeout (open.ts).\nconst SIDECAR_FILENAME = 'lock-wait.json';\nconst FLOOR_MS = 5_000;\nconst CEILING_MS = 600_000;\n\nfunction sidecarPath(baseDir: string): string {\n  return join(baseDir, STATE_DIR, SIDECAR_FILENAME);\n}\n\nfunction readRecordedMs(baseDir: string): number {\n  try {\n    const raw = JSON.parse(readFileSync(sidecarPath(baseDir), 'utf8'));\n    return typeof raw.reconcile_max_ms === 'number' ? raw.reconcile_max_ms : 0;\n  } catch {\n    return 0;\n  }\n}\n\n// Called after every reconcile's write transaction, every store alike, so a fresh duckdb/turso\n// tree starts building this the first time anything writes.\nexport function recordLockWaitMs(baseDir: string, ms: number): void {\n  if (ms <= readRecordedMs(baseDir)) return;\n  try {\n    writeFileSync(sidecarPath(baseDir), JSON.stringify({ reconcile_max_ms: ms }));\n  } catch {\n    // Best-effort: a write failure here only narrows the next waiter's budget back to the floor.\n  }\n}\n\n// 3x the largest reconcile this tree has recorded, floored and capped -- a waiter outlasts a\n// real reconcile and still fails behind a hung one.\nexport function lockWaitBudgetMs(baseDir: string): number {\n  return Math.min(Math.max(FLOOR_MS, 3 * readRecordedMs(baseDir)), CEILING_MS);\n}\n"],"names":["readFileSync","writeFileSync","join","STATE_DIR","SIDECAR_FILENAME","FLOOR_MS","CEILING_MS","sidecarPath","baseDir","readRecordedMs","raw","JSON","parse","reconcile_max_ms","recordLockWaitMs","ms","stringify","lockWaitBudgetMs","Math","min","max"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,QAAQ,UAAU;AACtD,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,oGAAoG;AACpG,yGAAyG;AACzG,MAAMC,mBAAmB;AACzB,MAAMC,WAAW;AACjB,MAAMC,aAAa;AAEnB,SAASC,YAAYC,OAAe;IAClC,OAAON,KAAKM,SAASL,WAAWC;AAClC;AAEA,SAASK,eAAeD,OAAe;IACrC,IAAI;QACF,MAAME,MAAMC,KAAKC,KAAK,CAACZ,aAAaO,YAAYC,UAAU;QAC1D,OAAO,OAAOE,IAAIG,gBAAgB,KAAK,WAAWH,IAAIG,gBAAgB,GAAG;IAC3E,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,+FAA+F;AAC/F,4DAA4D;AAC5D,OAAO,SAASC,iBAAiBN,OAAe,EAAEO,EAAU;IAC1D,IAAIA,MAAMN,eAAeD,UAAU;IACnC,IAAI;QACFP,cAAcM,YAAYC,UAAUG,KAAKK,SAAS,CAAC;YAAEH,kBAAkBE;QAAG;IAC5E,EAAE,OAAM;IACN,6FAA6F;IAC/F;AACF;AAEA,6FAA6F;AAC7F,oDAAoD;AACpD,OAAO,SAASE,iBAAiBT,OAAe;IAC9C,OAAOU,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAACf,UAAU,IAAII,eAAeD,WAAWF;AACnE"}

package/dist/esm/store/open.d.ts

@@ -1,5 +1,5 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { Builder } from './builder.js';
+import type { Stages } from './stages.js';
 import type { OpenDialect, Store } from './types.js';
 export interface OpenResult {
     store: Store;
@@ -7,9 +7,7 @@ export interface OpenResult {
     dbPath: string;
     parsed: number;
     warnings: string[];
+    stages: Stages;
 }
 export declare function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult>;
-export declare function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & {
-    builder: Builder;
-}>;
 export declare function docCount(store: Store): Promise<number>;

package/dist/esm/store/open.js

@@ -8,17 +8,16 @@ import { createBuilder } from './builder.js';
 import { clearCache } from './cache.js';
 import { classifyEmbedChange } from './embed-scope.js';
 import { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.js';
+import { lockWaitBudgetMs } from './lock-wait.js';
 import { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.js';
 import { getMeta, setMeta } from './shared.js';
 import { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.js';
-// duckdb and turso hold the cache file for their connection's whole life, not for a transaction,
-// so a second command waits on the first command finishing. Bounded rather than open-ended: this
-// clears a warm query's collision (~100ms) and still fails loudly behind a cold build instead of
-// looking hung.
-const LOCK_RETRY_MS = 5000;
+// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.
+// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.
 const LOCK_POLL_MS = 50;
 async function connectUnlocked(dbPath, cfg, dialect) {
-    const deadline = Date.now() + LOCK_RETRY_MS;
+    const budgetMs = lockWaitBudgetMs(cfg.baseDir);
+    const deadline = Date.now() + budgetMs;
     for(;;){
         try {
             return await dialect.connect(dbPath, cfg);
@@ -26,13 +25,13 @@ async function connectUnlocked(dbPath, cfg, dialect) {
             var _dialect_isLocked;
             if (!((_dialect_isLocked = dialect.isLocked) === null || _dialect_isLocked === void 0 ? void 0 : _dialect_isLocked.call(dialect, err))) throw err;
             if (Date.now() >= deadline) {
-                throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${LOCK_RETRY_MS / 1000}s; wait for that command to finish, or set "store" to "sqlite" in sense.config.json, which serves concurrent commands`);
+                throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set "store" to "sqlite" in sense.config.json, which serves concurrent commands`);
             }
             await new Promise((resolve)=>setTimeout(resolve, LOCK_POLL_MS));
         }
     }
 }
-async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
+async function connectWithDialect(cfg, dialect) {
     const stateDir = join(cfg.baseDir, STATE_DIR);
     mkdirSync(stateDir, {
         recursive: true
@@ -42,8 +41,8 @@ async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
     // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on
     // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.
     let closed = false;
-    // Created here, not inside reconcile(): its pool must survive across every build() call this
-    // connection sees (a watcher's repeated ticks), not just the one below.
+    // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool
+    // with the build() call that follows them.
     const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);
     try {
         await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
@@ -68,7 +67,7 @@ async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
                 closed = true;
                 await dialect.close(handle);
                 clearCache(cfg);
-                return connectWithDialect(cfg, dialect, keepBuilderOpen);
+                return connectWithDialect(cfg, dialect);
             }
             const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
             // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed
@@ -117,7 +116,7 @@ async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
                 closed = true;
                 await dialect.close(handle);
                 clearCache(cfg);
-                return connectWithDialect(cfg, dialect, keepBuilderOpen);
+                return connectWithDialect(cfg, dialect);
             }
         }
         await dialect.ensureSchema(handle, conn, cfg);
@@ -133,9 +132,9 @@ async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
         // feature-toggle-only changed-key set).
         const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;
         const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;
-        const { parsed, warnings } = await builder.build(forcedPaths);
-        // A one-shot open is done reconciling for good; a watcher keeps `builder` alive across its run.
-        if (!keepBuilderOpen) await builder.close();
+        const { parsed, warnings, stages } = await builder.build(forcedPaths);
+        // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).
+        await builder.close();
         return {
             handle,
             conn,
@@ -143,7 +142,7 @@ async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
             dbPath,
             parsed: parsed + embedParsed + featureParsed,
             warnings,
-            builder
+            stages
         };
     } catch (err) {
         await builder.close();
@@ -154,31 +153,17 @@ async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
 // A store's open(): connects (see connectWithDialect above), then wraps the resulting connection
 // in the Store interface. The builder's pool closes right after the initial build.
 export async function openWithDialect(cfg, dialect) {
-    const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connectWithDialect(cfg, dialect, false);
+    const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);
     const store = dialect.createStore(handle, conn, resolvedCfg);
     // reconcile ran before this object existed, so its chunk text is keyed by the connection.
     rekeyChunkText(conn, store);
-    return {
-        store,
-        cfg: resolvedCfg,
-        dbPath,
-        parsed,
-        warnings
-    };
-}
-// Same connect+build as openWithDialect, but keeps the builder's pool alive and hands it back
-// alongside the store, for a caller (a watcher) that reconciles repeatedly on this connection.
-export async function openWithBuilder(cfg, dialect) {
-    const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, builder } = await connectWithDialect(cfg, dialect, true);
-    const store = dialect.createStore(handle, conn, resolvedCfg);
-    rekeyChunkText(conn, store);
     return {
         store,
         cfg: resolvedCfg,
         dbPath,
         parsed,
         warnings,
-        builder
+        stages
     };
 }
 export async function docCount(store) {

package/dist/esm/store/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { Builder } from './builder.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Closed already unless `keepBuilderOpen` was set: a one-shot open reconciles once and is done\n  // with it, a watcher keeps calling build() on the same instance across its whole run.\n  builder: Builder;\n}\n\n// duckdb and turso hold the cache file for their connection's whole life, not for a transaction,\n// so a second command waits on the first command finishing. Bounded rather than open-ended: this\n// clears a warm query's collision (~100ms) and still fails loudly behind a cold build instead of\n// looking hung.\nconst LOCK_RETRY_MS = 5_000;\nconst LOCK_POLL_MS = 50;\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const deadline = Date.now() + LOCK_RETRY_MS;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${LOCK_RETRY_MS / 1000}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>, keepBuilderOpen: boolean): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): its pool must survive across every build() call this\n  // connection sees (a watcher's repeated ticks), not just the one below.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings } = await builder.build(forcedPaths);\n    // A one-shot open is done reconciling for good; a watcher keeps `builder` alive across its run.\n    if (!keepBuilderOpen) await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, builder };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connectWithDialect(cfg, dialect, false);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\n// Same connect+build as openWithDialect, but keeps the builder's pool alive and hands it back\n// alongside the store, for a caller (a watcher) that reconciles repeatedly on this connection.\nexport async function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & { builder: Builder }> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, builder } = await connectWithDialect(cfg, dialect, true);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, builder };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_RETRY_MS","LOCK_POLL_MS","connectUnlocked","dbPath","cfg","dialect","deadline","Date","now","connect","err","isLocked","store","Promise","resolve","setTimeout","connectWithDialect","keepBuilderOpen","stateDir","baseDir","recursive","filename","handle","conn","closed","builder","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","Math","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","build","openWithDialect","resolvedCfg","createStore","openWithBuilder","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA4B3F,iGAAiG;AACjG,iGAAiG;AACjG,iGAAiG;AACjG,gBAAgB;AAChB,MAAMC,gBAAgB;AACtB,MAAMC,eAAe;AAErB,eAAeC,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;IACtG,MAAMC,WAAWC,KAAKC,GAAG,KAAKR;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMK,QAAQI,OAAO,CAACN,QAAQC;QACvC,EAAE,OAAOM,KAAK;gBACPL;YAAL,IAAI,GAACA,oBAAAA,QAAQM,QAAQ,cAAhBN,wCAAAA,uBAAAA,SAAmBK,OAAe,MAAMA;YAC7C,IAAIH,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAIpB,WAAW,cAAc,CAAC,2CAA2C,EAAEkB,IAAIQ,KAAK,CAAC,QAAQ,EAAET,OAAO,gCAAgC,EAAEH,gBAAgB,KAAK,qHAAqH,CAAC;YAC3R;YACA,MAAM,IAAIa,QAAQ,CAACC,UAAYC,WAAWD,SAASb;QACrD;IACF;AACF;AAEA,eAAee,mBAA2BZ,GAAmB,EAAEC,OAA4B,EAAEY,eAAwB;IACnH,MAAMC,WAAWpC,KAAKsB,IAAIe,OAAO,EAAEnC;IACnCH,UAAUqC,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMjB,SAASrB,KAAKoC,UAAUb,QAAQgB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAMrB,gBAAgBC,QAAQC,KAAKC;IAC5D,gGAAgG;IAChG,kGAAkG;IAClG,IAAImB,SAAS;IACb,6FAA6F;IAC7F,wEAAwE;IACxE,MAAMC,UAAUrC,cAAcmC,MAAMnB,KAAKA,IAAIe,OAAO,EAAEd,QAAQqB,gBAAgB;IAC9E,IAAI;QACF,MAAMH,KAAKI,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMjC,QAAQ4B,MAAM;QACpC,MAAMM,WAAW,MAAMlC,QAAQ4B,MAAM;QACrC,MAAMO,eAAe/C,iBAAiBqB,KAAKjB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAI4C;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAYvB,QAAQ6B,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;YAC/G,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYvB,QAAQ6B,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFZ,SAAS;gBACT,MAAMnB,QAAQgC,KAAK,CAACf;gBACpBjC,WAAWe;gBACX,OAAOY,mBAAmBZ,KAAKC,SAASY;YAC1C;YACA,MAAMqB,cAAczC,qBAAqBgC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAe7C,mBAAmB4C,eAAe7C,kBAAkBW,KAAKA,IAAIe,OAAO,EAAEU,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YAC1H,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAWpD,oBAAoBuC,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAUnD,oBAAoB8C,eAAe/C,uBAAuBsC,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAY5C,qBAAqB+B,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAMxC,QAAQ2B,MAAM,YAAYO;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAM5C,QAAQ2B,MAAM,YAAYO;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAU7C,cAAc8B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMhD,QAAQ2B,MAAM,YAAYO;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAU7C,cAAc8B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMhD,QAAQ2B,MAAM,YAAYO;YAClC,OAAO;gBACL,MAAMc,UAAU7C,cAAc8B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpEpB,SAAS;gBACT,MAAMnB,QAAQgC,KAAK,CAACf;gBACpBjC,WAAWe;gBACX,OAAOY,mBAAmBZ,KAAKC,SAASY;YAC1C;QACF;QAEA,MAAMZ,QAAQwC,YAAY,CAACvB,QAAQC,MAAMnB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIC,QAAQyC,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAMrD,QAAQ4B,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMlB,QAAQyC,qBAAqB,CAACxB,QAAQC,MAAM0B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMK,cAAcpB,kBAAkB,AAAC,CAAA,MAAMP,QAAQ4B,UAAU,CAACrB,gBAAe,EAAGsB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBtB,iBAAiB,AAAC,CAAA,MAAMR,QAAQ+B,kBAAkB,CAACvB,eAAc,EAAGqB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAE,GAAG,MAAMhC,QAAQiC,KAAK,CAAC3B;QACjD,gGAAgG;QAChG,IAAI,CAACd,iBAAiB,MAAMQ,QAAQY,KAAK;QAEzC,OAAO;YAAEf;YAAQC;YAAMnB;YAAKD;YAAQmD,QAAQA,SAASF,cAAcG;YAAeE;YAAUhC;QAAQ;IACtG,EAAE,OAAOf,KAAK;QACZ,MAAMe,QAAQY,KAAK;QACnB,IAAI,CAACb,QAAQ,MAAMnB,QAAQgC,KAAK,CAACf;QACjC,MAAMZ;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeiD,gBAAwBvD,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEiB,MAAM,EAAEC,IAAI,EAAEnB,KAAKwD,WAAW,EAAEzD,MAAM,EAAEmD,MAAM,EAAEG,QAAQ,EAAE,GAAG,MAAMzC,mBAAmBZ,KAAKC,SAAS;IAC5G,MAAMO,QAAQP,QAAQwD,WAAW,CAACvC,QAAQC,MAAMqC;IAChD,0FAA0F;IAC1F3E,eAAesC,MAAMX;IACrB,OAAO;QAAEA;QAAOR,KAAKwD;QAAazD;QAAQmD;QAAQG;IAAS;AAC7D;AAEA,8FAA8F;AAC9F,+FAA+F;AAC/F,OAAO,eAAeK,gBAAwB1D,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEiB,MAAM,EAAEC,IAAI,EAAEnB,KAAKwD,WAAW,EAAEzD,MAAM,EAAEmD,MAAM,EAAEG,QAAQ,EAAEhC,OAAO,EAAE,GAAG,MAAMT,mBAAmBZ,KAAKC,SAAS;IACrH,MAAMO,QAAQP,QAAQwD,WAAW,CAACvC,QAAQC,MAAMqC;IAChD3E,eAAesC,MAAMX;IACrB,OAAO;QAAEA;QAAOR,KAAKwD;QAAazD;QAAQmD;QAAQG;QAAUhC;IAAQ;AACtE;AAEA,OAAO,eAAesC,SAASnD,KAAY;IACzC,MAAMoD,OAAO,MAAMpD,MAAMqD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(cfg.baseDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","lockWaitBudgetMs","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_POLL_MS","connectUnlocked","dbPath","cfg","dialect","budgetMs","baseDir","deadline","Date","now","connect","err","isLocked","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","stateDir","recursive","filename","handle","conn","closed","builder","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","stages","build","openWithDialect","resolvedCfg","createStore","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA8B3F,iGAAiG;AACjG,oFAAoF;AACpF,MAAMC,eAAe;AAErB,eAAeC,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;IACtG,MAAMC,WAAWb,iBAAiBW,IAAIG,OAAO;IAC7C,MAAMC,WAAWC,KAAKC,GAAG,KAAKJ;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMD,QAAQM,OAAO,CAACR,QAAQC;QACvC,EAAE,OAAOQ,KAAK;gBACPP;YAAL,IAAI,GAACA,oBAAAA,QAAQQ,QAAQ,cAAhBR,wCAAAA,uBAAAA,SAAmBO,OAAe,MAAMA;YAC7C,IAAIH,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAItB,WAAW,cAAc,CAAC,2CAA2C,EAAEkB,IAAIU,KAAK,CAAC,QAAQ,EAAEX,OAAO,gCAAgC,EAAEY,KAAKC,KAAK,CAACV,WAAW,MAAM,qHAAqH,CAAC;YAClS;YACA,MAAM,IAAIW,QAAQ,CAACC,UAAYC,WAAWD,SAASjB;QACrD;IACF;AACF;AAEA,eAAemB,mBAA2BhB,GAAmB,EAAEC,OAA4B;IACzF,MAAMgB,WAAWvC,KAAKsB,IAAIG,OAAO,EAAEvB;IACnCH,UAAUwC,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMnB,SAASrB,KAAKuC,UAAUhB,QAAQkB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAMvB,gBAAgBC,QAAQC,KAAKC;IAC5D,gGAAgG;IAChG,kGAAkG;IAClG,IAAIqB,SAAS;IACb,+FAA+F;IAC/F,2CAA2C;IAC3C,MAAMC,UAAUvC,cAAcqC,MAAMrB,KAAKA,IAAIG,OAAO,EAAEF,QAAQuB,gBAAgB;IAC9E,IAAI;QACF,MAAMH,KAAKI,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMlC,QAAQ6B,MAAM;QACpC,MAAMM,WAAW,MAAMnC,QAAQ6B,MAAM;QACrC,MAAMO,eAAejD,iBAAiBqB,KAAKjB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAI8C;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAYzB,QAAQ+B,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;YAC/G,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYzB,QAAQ+B,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFZ,SAAS;gBACT,MAAMrB,QAAQkC,KAAK,CAACf;gBACpBnC,WAAWe;gBACX,OAAOgB,mBAAmBhB,KAAKC;YACjC;YACA,MAAMmC,cAAc1C,qBAAqBiC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAe9C,mBAAmB6C,eAAe9C,kBAAkBU,KAAKA,IAAIG,OAAO,EAAEwB,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YAC1H,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAWtD,oBAAoByC,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAUrD,oBAAoBgD,eAAejD,uBAAuBwC,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAY7C,qBAAqBgC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAMzC,QAAQ4B,MAAM,YAAYO;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAM7C,QAAQ4B,MAAM,YAAYO;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAU9C,cAAc+B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMjD,QAAQ4B,MAAM,YAAYO;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAU9C,cAAc+B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMjD,QAAQ4B,MAAM,YAAYO;YAClC,OAAO;gBACL,MAAMc,UAAU9C,cAAc+B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpEpB,SAAS;gBACT,MAAMrB,QAAQkC,KAAK,CAACf;gBACpBnC,WAAWe;gBACX,OAAOgB,mBAAmBhB,KAAKC;YACjC;QACF;QAEA,MAAMA,QAAQ0C,YAAY,CAACvB,QAAQC,MAAMrB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIC,QAAQ2C,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAMtD,QAAQ6B,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMpB,QAAQ2C,qBAAqB,CAACxB,QAAQC,MAAMV,KAAKoC,GAAG,CAACpC,KAAKqC,GAAG,CAAC,OAAO,IAAIH,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMI,cAAcnB,kBAAkB,AAAC,CAAA,MAAMP,QAAQ2B,UAAU,CAACpB,gBAAe,EAAGqB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBrB,iBAAiB,AAAC,CAAA,MAAMR,QAAQ8B,kBAAkB,CAACtB,eAAc,EAAGoB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMhC,QAAQiC,KAAK,CAAC3B;QACzD,uIAAuI;QACvI,MAAMN,QAAQY,KAAK;QAEnB,OAAO;YAAEf;YAAQC;YAAMrB;YAAKD;YAAQoD,QAAQA,SAASF,cAAcG;YAAeE;YAAUC;QAAO;IACrG,EAAE,OAAO/C,KAAK;QACZ,MAAMe,QAAQY,KAAK;QACnB,IAAI,CAACb,QAAQ,MAAMrB,QAAQkC,KAAK,CAACf;QACjC,MAAMZ;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeiD,gBAAwBzD,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEmB,MAAM,EAAEC,IAAI,EAAErB,KAAK0D,WAAW,EAAE3D,MAAM,EAAEoD,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMvC,mBAAmBhB,KAAKC;IAC3G,MAAMS,QAAQT,QAAQ0D,WAAW,CAACvC,QAAQC,MAAMqC;IAChD,0FAA0F;IAC1F7E,eAAewC,MAAMX;IACrB,OAAO;QAAEA;QAAOV,KAAK0D;QAAa3D;QAAQoD;QAAQG;QAAUC;IAAO;AACrE;AAEA,OAAO,eAAeK,SAASlD,KAAY;IACzC,MAAMmD,OAAO,MAAMnD,MAAMoD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}

package/dist/esm/store/reconcile.d.ts

@@ -1,8 +1,10 @@
 import type { Config } from '../config/index.js';
 import type { ParsePool } from '../scan/pool.js';
+import { type Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export declare const CORE_FRONTMATTER_COLUMNS: Set<string>;
 export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{
     parsed: number;
     warnings: string[];
+    stages: Stages;
 }>;

package/dist/esm/store/reconcile.js

@@ -2,7 +2,9 @@ import { activeFeatures } from '../features/index.js';
 import { progress } from '../output/progress.js';
 import { listFiles, RESERVED_COLUMNS } from '../scan/index.js';
 import { reparseFiles } from '../scan/reparse.js';
+import { recordLockWaitMs } from './lock-wait.js';
 import { getColumns, quoteIdent } from './shared.js';
+import { featureStage, stageRecorder } from './stages.js';
 import { withTransaction } from './transaction.js';
 // One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect
 // (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,
@@ -19,10 +21,16 @@ export const CORE_FRONTMATTER_COLUMNS = new Set([
     '_parse_error'
 ]);
 export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
-    const files = listFiles(cfg, baseDir);
+    const start = process.hrtime.bigint();
+    const features = activeFeatures(cfg);
+    const stages = stageRecorder(features.map((f)=>f.name));
+    const elapsed = ()=>Number(process.hrtime.bigint() - start) / 1e6;
+    const files = await stages.time('list', ()=>listFiles(cfg, baseDir));
     const currentSet = new Set(files.map((f)=>f.relPath));
-    const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
-    const existingRows = await existingStmt.all();
+    const existingRows = await stages.time('existing', async ()=>{
+        const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
+        return await existingStmt.all();
+    });
     const existing = new Map(existingRows.map((r)=>[
             r.path,
             r
@@ -40,16 +48,17 @@ export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths)
     });
     if (vanished.length === 0 && toReparse.length === 0) return {
         parsed: 0,
-        warnings: []
+        warnings: [],
+        stages: stages.take(elapsed(), 0)
     };
-    const features = activeFeatures(cfg);
     const seenColumns = await getColumns(conn);
     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
     // reconciles stay silent (progress() has a threshold).
     const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick, {
-        pool
-    });
+    // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.
+    const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', ()=>reparseFiles(toReparse, features, cfg, seenColumns, report.tick, {
+            pool
+        }));
     report.finish();
     for (const col of newColumns)seenColumns.add(col);
     const allColumns = [
@@ -81,27 +90,26 @@ export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths)
     ];
     const txStart = Date.now();
     await withTransaction(conn, async ()=>{
-        var _feature_remove, _feature_afterReconcile;
         // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a
         // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.
         const present = await getColumns(conn);
         const missingColumns = newColumns.filter((col)=>!present.has(col));
-        if (missingColumns.length > 0) await dialect.addColumns(conn, missingColumns);
+        if (missingColumns.length > 0) await stages.time('alter', ()=>dialect.addColumns(conn, missingColumns));
         // Revalidated once the lock is held, before this process's own frontmatter upsert below:
         // `added` came from a path read taken before this transaction's lock, so a path still
         // called "added" here may already have a content row from a concurrent reconcile that
         // committed while this one waited. One SELECT, not a DELETE per added file -- with no
         // contention it returns the same set `existing` already ruled out, so nothing extra clears.
         let contentTouched = touched;
-        if (added.length > 0) {
+        if (added.length > 0) contentTouched = await stages.time('added-recheck', async ()=>{
             const currentPathsStmt = await conn.prepare('SELECT "path" FROM frontmatter');
             const currentPaths = new Set((await currentPathsStmt.all()).map((r)=>r.path));
             const staleAdded = added.filter((p)=>currentPaths.has(p));
-            if (staleAdded.length > 0) contentTouched = [
+            return staleAdded.length > 0 ? [
                 ...touched,
                 ...staleAdded
-            ];
-        }
+            ] : touched;
+        });
         if (parsedDocs.length > 0) {
             const toRow = (doc)=>writableColumns.map((col)=>{
                     var _doc_data_col;
@@ -115,52 +123,72 @@ export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths)
                 });
             // A path in `added` has no existing frontmatter row, so it can never conflict: where a
             // dialect offers a faster append-only path (insertNew), only the rest go through the upsert.
-            if (dialect.insertNew) {
-                const newDocs = parsedDocs.filter((d)=>addedSet.has(d.relPath));
-                const updateDocs = parsedDocs.filter((d)=>!addedSet.has(d.relPath));
-                if (newDocs.length > 0) await dialect.insertNew(conn, 'frontmatter', writableColumns, newDocs.map(toRow));
-                if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));
-            } else {
-                await conn.runBatch(insertSql, parsedDocs.map(toRow));
-            }
+            await stages.time('fm-upsert', async ()=>{
+                if (dialect.insertNew) {
+                    const newDocs = parsedDocs.filter((d)=>addedSet.has(d.relPath));
+                    const updateDocs = parsedDocs.filter((d)=>!addedSet.has(d.relPath));
+                    if (newDocs.length > 0) await dialect.insertNew(conn, 'frontmatter', writableColumns, newDocs.map(toRow));
+                    if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));
+                } else {
+                    await conn.runBatch(insertSql, parsedDocs.map(toRow));
+                }
+            });
         }
         // After the upsert, so every doc already has the frontmatter row sqlite's content rowid
         // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.
-        await dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg);
+        await stages.time('text-index', ()=>dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));
         // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
         // new docs have nothing to clear, which keeps cold builds linear.
-        if (touched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map((p)=>[
-                p
-            ]));
-        const presetRows = [];
-        for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
-            doc.relPath,
-            presetName
-        ]);
-        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken
-        // before this transaction's lock, so a path this process calls "added" can already have
-        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.
-        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows);
+        await stages.time('presets', async ()=>{
+            if (touched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map((p)=>[
+                    p
+                ]));
+            const presetRows = [];
+            for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
+                doc.relPath,
+                presetName
+            ]);
+            // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken
+            // before this transaction's lock, so a path this process calls "added" can already have
+            // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.
+            if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows);
+        });
         // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's
         // node set, so a lingering vanished row would dilute rank mass across every surviving note.
-        if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
-                p
-            ]));
-        if (touched.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta));
+        if (vanished.length > 0) await stages.time('vanished', ()=>conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
+                    p
+                ])));
+        // Timed per feature per hook, so link resolution and PageRank are named stages without
+        // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.
+        if (touched.length > 0) for (const feature of features)await stages.time(featureStage(feature.name, 'remove'), ()=>{
+            var _feature_remove;
+            return (_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta);
+        });
         for (const feature of features){
-            var _feature_store;
             const docsForFeature = parsedDocs.map((doc)=>({
                     path: doc.relPath,
                     extracted: doc.extracted[feature.name]
                 }));
-            await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
+            await stages.time(featureStage(feature.name, 'store'), ()=>{
+                var _feature_store;
+                return (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta);
+            });
         }
-        for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
+        for (const feature of features)await stages.time(featureStage(feature.name, 'after'), ()=>{
+            var _feature_afterReconcile;
+            return (_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta);
+        });
     }, dialect.beginMode());
     const durationMs = Date.now() - txStart;
-    if (dialect.recordDuration) await dialect.recordDuration(conn, durationMs);
+    // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.
+    recordLockWaitMs(baseDir, durationMs);
+    if (dialect.recordDuration) await stages.time('meta', ()=>{
+        var _dialect_recordDuration;
+        return (_dialect_recordDuration = dialect.recordDuration) === null || _dialect_recordDuration === void 0 ? void 0 : _dialect_recordDuration.call(dialect, conn, durationMs);
+    });
     return {
         parsed: parsedDocs.length,
-        warnings
+        warnings,
+        stages: stages.take(elapsed(), durationMs, workerParseMs)
     };
 }