sensemaking 0.19.2 → 0.21.0

package/dist/esm/store/open.js

@@ -0,0 +1,187 @@
+import { mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { featureSignature, STATE_DIR } from '../config/index.js';
+import { rekeyChunkText } from '../embed/handoff.js';
+import { SenseError } from '../errors.js';
+import { FEATURES } from '../features/index.js';
+import { createBuilder } from './builder.js';
+import { clearCache } from './cache.js';
+import { classifyEmbedChange } from './embed-scope.js';
+import { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.js';
+import { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.js';
+import { getMeta, setMeta } from './shared.js';
+import { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.js';
+// duckdb and turso hold the cache file for their connection's whole life, not for a transaction,
+// so a second command waits on the first command finishing. Bounded rather than open-ended: this
+// clears a warm query's collision (~100ms) and still fails loudly behind a cold build instead of
+// looking hung.
+const LOCK_RETRY_MS = 5000;
+const LOCK_POLL_MS = 50;
+async function connectUnlocked(dbPath, cfg, dialect) {
+    const deadline = Date.now() + LOCK_RETRY_MS;
+    for(;;){
+        try {
+            return await dialect.connect(dbPath, cfg);
+        } catch (err) {
+            var _dialect_isLocked;
+            if (!((_dialect_isLocked = dialect.isLocked) === null || _dialect_isLocked === void 0 ? void 0 : _dialect_isLocked.call(dialect, err))) throw err;
+            if (Date.now() >= deadline) {
+                throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${LOCK_RETRY_MS / 1000}s; wait for that command to finish, or set "store" to "sqlite" in sense.config.json, which serves concurrent commands`);
+            }
+            await new Promise((resolve)=>setTimeout(resolve, LOCK_POLL_MS));
+        }
+    }
+}
+async function connectWithDialect(cfg, dialect, keepBuilderOpen) {
+    const stateDir = join(cfg.baseDir, STATE_DIR);
+    mkdirSync(stateDir, {
+        recursive: true
+    });
+    const dbPath = join(stateDir, dialect.filename);
+    const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);
+    // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on
+    // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.
+    let closed = false;
+    // Created here, not inside reconcile(): its pool must survive across every build() call this
+    // connection sees (a watcher's repeated ticks), not just the one below.
+    const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);
+    try {
+        await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
+        // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an
+        // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).
+        const version = await getMeta(conn, 'schema_version');
+        const features = await getMeta(conn, 'features');
+        const wantFeatures = featureSignature(cfg, FEATURES);
+        // Set only by the preset-membership branch below; reconcile treats these paths as touched
+        // despite an unchanged stamp, for this one build() call.
+        let forcedPaths;
+        // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.
+        let embedInvalidate;
+        // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.
+        let featureToggles;
+        if (version !== null && version !== dialect.schemaVersion || features !== null && features !== wantFeatures) {
+            // Indexing derives from presets, so a config edit rebuilding the cache must say so and
+            // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
+            if (version !== null && version !== dialect.schemaVersion) {
+                console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
+                // Close before clearCache deletes the files out from under the still-open handle.
+                closed = true;
+                await dialect.close(handle);
+                clearCache(cfg);
+                return connectWithDialect(cfg, dialect, keepBuilderOpen);
+            }
+            const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
+            // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed
+            // preset's stored segment can't be parsed back into old include/exclude/vectors -- either
+            // way the branch below is skipped and the full rebuild stays the fallback.
+            const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features !== null && features !== void 0 ? features : '', changedKeys) : null;
+            // Null unless exactly one segment changed and it decomposes into a recognised embed-only
+            // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.
+            const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features !== null && features !== void 0 ? features : '', wantFeatures) : null;
+            // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed
+            // feature's stored segment can't be parsed back into old/new on-off -- either way the branch
+            // below is skipped and the full rebuild stays the fallback.
+            const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features !== null && features !== void 0 ? features : '', wantFeatures, changedKeys) : null;
+            if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
+                // First sight of a resolved weight identity: the model itself hasn't changed, so
+                // adopt it into meta with no rebuild and no re-embed.
+                console.error("sense: recorded the embedding model's resolved identity; vectors are unaffected");
+                await setMeta(conn, 'features', wantFeatures);
+            } else if (embedKind !== null) {
+                // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither
+                // case needs any file reparsed for any feature but embed itself.
+                embedInvalidate = embedKind;
+                console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');
+                await setMeta(conn, 'features', wantFeatures);
+            } else if (presetForced !== null) {
+                // Membership only: which files a preset covers moved, not a file's own content, a
+                // global feature, or the embed model. reconcile() already knows how to
+                // add, update, and remove a file correctly (every cross-feature cascade included); it
+                // just needs telling which unchanged files to treat as touched (forcedPaths, above).
+                forcedPaths = presetForced;
+                const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
+                console.error(`sense: config change (${changed}) reparses only the files it affects`);
+                await setMeta(conn, 'features', wantFeatures);
+            } else if (toggles !== null) {
+                // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's
+                // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a
+                // feature turning back on, fully re-derives across every indexed file rather than
+                // trusting rows written before or during that window.
+                featureToggles = toggles;
+                const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
+                console.error(`sense: config change (${changed}) reparses only the feature it affects`);
+                await setMeta(conn, 'features', wantFeatures);
+            } else {
+                const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
+                console.error(`sense: config change (${changed}) rebuilds the index`);
+                closed = true;
+                await dialect.close(handle);
+                clearCache(cfg);
+                return connectWithDialect(cfg, dialect, keepBuilderOpen);
+            }
+        }
+        await dialect.ensureSchema(handle, conn, cfg);
+        // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
+        // Installed before build() -- that call is the one that races a watcher's transaction.
+        if (dialect.setDerivedBusyTimeout) {
+            var _ref;
+            const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
+            await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600000));
+        }
+        // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same
+        // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or
+        // feature-toggle-only changed-key set).
+        const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;
+        const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;
+        const { parsed, warnings } = await builder.build(forcedPaths);
+        // A one-shot open is done reconciling for good; a watcher keeps `builder` alive across its run.
+        if (!keepBuilderOpen) await builder.close();
+        return {
+            handle,
+            conn,
+            cfg,
+            dbPath,
+            parsed: parsed + embedParsed + featureParsed,
+            warnings,
+            builder
+        };
+    } catch (err) {
+        await builder.close();
+        if (!closed) await dialect.close(handle);
+        throw err;
+    }
+}
+// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection
+// in the Store interface. The builder's pool closes right after the initial build.
+export async function openWithDialect(cfg, dialect) {
+    const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connectWithDialect(cfg, dialect, false);
+    const store = dialect.createStore(handle, conn, resolvedCfg);
+    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+    rekeyChunkText(conn, store);
+    return {
+        store,
+        cfg: resolvedCfg,
+        dbPath,
+        parsed,
+        warnings
+    };
+}
+// Same connect+build as openWithDialect, but keeps the builder's pool alive and hands it back
+// alongside the store, for a caller (a watcher) that reconciles repeatedly on this connection.
+export async function openWithBuilder(cfg, dialect) {
+    const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, builder } = await connectWithDialect(cfg, dialect, true);
+    const store = dialect.createStore(handle, conn, resolvedCfg);
+    rekeyChunkText(conn, store);
+    return {
+        store,
+        cfg: resolvedCfg,
+        dbPath,
+        parsed,
+        warnings,
+        builder
+    };
+}
+export async function docCount(store) {
+    const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');
+    return (await stmt.get()).n;
+}

package/dist/esm/store/open.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { Builder } from './builder.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Closed already unless `keepBuilderOpen` was set: a one-shot open reconciles once and is done\n  // with it, a watcher keeps calling build() on the same instance across its whole run.\n  builder: Builder;\n}\n\n// duckdb and turso hold the cache file for their connection's whole life, not for a transaction,\n// so a second command waits on the first command finishing. Bounded rather than open-ended: this\n// clears a warm query's collision (~100ms) and still fails loudly behind a cold build instead of\n// looking hung.\nconst LOCK_RETRY_MS = 5_000;\nconst LOCK_POLL_MS = 50;\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const deadline = Date.now() + LOCK_RETRY_MS;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${LOCK_RETRY_MS / 1000}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>, keepBuilderOpen: boolean): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): its pool must survive across every build() call this\n  // connection sees (a watcher's repeated ticks), not just the one below.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings } = await builder.build(forcedPaths);\n    // A one-shot open is done reconciling for good; a watcher keeps `builder` alive across its run.\n    if (!keepBuilderOpen) await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, builder };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connectWithDialect(cfg, dialect, false);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\n// Same connect+build as openWithDialect, but keeps the builder's pool alive and hands it back\n// alongside the store, for a caller (a watcher) that reconciles repeatedly on this connection.\nexport async function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & { builder: Builder }> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, builder } = await connectWithDialect(cfg, dialect, true);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, builder };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_RETRY_MS","LOCK_POLL_MS","connectUnlocked","dbPath","cfg","dialect","deadline","Date","now","connect","err","isLocked","store","Promise","resolve","setTimeout","connectWithDialect","keepBuilderOpen","stateDir","baseDir","recursive","filename","handle","conn","closed","builder","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","Math","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","build","openWithDialect","resolvedCfg","createStore","openWithBuilder","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA4B3F,iGAAiG;AACjG,iGAAiG;AACjG,iGAAiG;AACjG,gBAAgB;AAChB,MAAMC,gBAAgB;AACtB,MAAMC,eAAe;AAErB,eAAeC,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;IACtG,MAAMC,WAAWC,KAAKC,GAAG,KAAKR;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMK,QAAQI,OAAO,CAACN,QAAQC;QACvC,EAAE,OAAOM,KAAK;gBACPL;YAAL,IAAI,GAACA,oBAAAA,QAAQM,QAAQ,cAAhBN,wCAAAA,uBAAAA,SAAmBK,OAAe,MAAMA;YAC7C,IAAIH,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAIpB,WAAW,cAAc,CAAC,2CAA2C,EAAEkB,IAAIQ,KAAK,CAAC,QAAQ,EAAET,OAAO,gCAAgC,EAAEH,gBAAgB,KAAK,qHAAqH,CAAC;YAC3R;YACA,MAAM,IAAIa,QAAQ,CAACC,UAAYC,WAAWD,SAASb;QACrD;IACF;AACF;AAEA,eAAee,mBAA2BZ,GAAmB,EAAEC,OAA4B,EAAEY,eAAwB;IACnH,MAAMC,WAAWpC,KAAKsB,IAAIe,OAAO,EAAEnC;IACnCH,UAAUqC,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMjB,SAASrB,KAAKoC,UAAUb,QAAQgB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAMrB,gBAAgBC,QAAQC,KAAKC;IAC5D,gGAAgG;IAChG,kGAAkG;IAClG,IAAImB,SAAS;IACb,6FAA6F;IAC7F,wEAAwE;IACxE,MAAMC,UAAUrC,cAAcmC,MAAMnB,KAAKA,IAAIe,OAAO,EAAEd,QAAQqB,gBAAgB;IAC9E,IAAI;QACF,MAAMH,KAAKI,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMjC,QAAQ4B,MAAM;QACpC,MAAMM,WAAW,MAAMlC,QAAQ4B,MAAM;QACrC,MAAMO,eAAe/C,iBAAiBqB,KAAKjB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAI4C;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAYvB,QAAQ6B,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;YAC/G,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYvB,QAAQ6B,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFZ,SAAS;gBACT,MAAMnB,QAAQgC,KAAK,CAACf;gBACpBjC,WAAWe;gBACX,OAAOY,mBAAmBZ,KAAKC,SAASY;YAC1C;YACA,MAAMqB,cAAczC,qBAAqBgC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAe7C,mBAAmB4C,eAAe7C,kBAAkBW,KAAKA,IAAIe,OAAO,EAAEU,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YAC1H,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAWpD,oBAAoBuC,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAUnD,oBAAoB8C,eAAe/C,uBAAuBsC,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAY5C,qBAAqB+B,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAMxC,QAAQ2B,MAAM,YAAYO;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAM5C,QAAQ2B,MAAM,YAAYO;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAU7C,cAAc8B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMhD,QAAQ2B,MAAM,YAAYO;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAU7C,cAAc8B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMhD,QAAQ2B,MAAM,YAAYO;YAClC,OAAO;gBACL,MAAMc,UAAU7C,cAAc8B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpEpB,SAAS;gBACT,MAAMnB,QAAQgC,KAAK,CAACf;gBACpBjC,WAAWe;gBACX,OAAOY,mBAAmBZ,KAAKC,SAASY;YAC1C;QACF;QAEA,MAAMZ,QAAQwC,YAAY,CAACvB,QAAQC,MAAMnB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIC,QAAQyC,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAMrD,QAAQ4B,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMlB,QAAQyC,qBAAqB,CAACxB,QAAQC,MAAM0B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMK,cAAcpB,kBAAkB,AAAC,CAAA,MAAMP,QAAQ4B,UAAU,CAACrB,gBAAe,EAAGsB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBtB,iBAAiB,AAAC,CAAA,MAAMR,QAAQ+B,kBAAkB,CAACvB,eAAc,EAAGqB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAE,GAAG,MAAMhC,QAAQiC,KAAK,CAAC3B;QACjD,gGAAgG;QAChG,IAAI,CAACd,iBAAiB,MAAMQ,QAAQY,KAAK;QAEzC,OAAO;YAAEf;YAAQC;YAAMnB;YAAKD;YAAQmD,QAAQA,SAASF,cAAcG;YAAeE;YAAUhC;QAAQ;IACtG,EAAE,OAAOf,KAAK;QACZ,MAAMe,QAAQY,KAAK;QACnB,IAAI,CAACb,QAAQ,MAAMnB,QAAQgC,KAAK,CAACf;QACjC,MAAMZ;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeiD,gBAAwBvD,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEiB,MAAM,EAAEC,IAAI,EAAEnB,KAAKwD,WAAW,EAAEzD,MAAM,EAAEmD,MAAM,EAAEG,QAAQ,EAAE,GAAG,MAAMzC,mBAAmBZ,KAAKC,SAAS;IAC5G,MAAMO,QAAQP,QAAQwD,WAAW,CAACvC,QAAQC,MAAMqC;IAChD,0FAA0F;IAC1F3E,eAAesC,MAAMX;IACrB,OAAO;QAAEA;QAAOR,KAAKwD;QAAazD;QAAQmD;QAAQG;IAAS;AAC7D;AAEA,8FAA8F;AAC9F,+FAA+F;AAC/F,OAAO,eAAeK,gBAAwB1D,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEiB,MAAM,EAAEC,IAAI,EAAEnB,KAAKwD,WAAW,EAAEzD,MAAM,EAAEmD,MAAM,EAAEG,QAAQ,EAAEhC,OAAO,EAAE,GAAG,MAAMT,mBAAmBZ,KAAKC,SAAS;IACrH,MAAMO,QAAQP,QAAQwD,WAAW,CAACvC,QAAQC,MAAMqC;IAChD3E,eAAesC,MAAMX;IACrB,OAAO;QAAEA;QAAOR,KAAKwD;QAAazD;QAAQmD;QAAQG;QAAUhC;IAAQ;AACtE;AAEA,OAAO,eAAesC,SAASnD,KAAY;IACzC,MAAMoD,OAAO,MAAMpD,MAAMqD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}

package/dist/esm/store/preset-scope.d.ts

@@ -0,0 +1,3 @@
+import type { Config } from '../config/index.js';
+export declare function isPresetOnlyChange(changedKeys: ReadonlySet<string>): boolean;
+export declare function forcedPresetPaths(cfg: Config, baseDir: string, before: string, changedKeys: ReadonlySet<string>): Set<string> | null;

package/dist/esm/store/preset-scope.js

@@ -0,0 +1,77 @@
+import { presetHasSignal } from '../config/index.js';
+import { listFiles } from '../scan/index.js';
+// A signature change open() can route to a narrow reparse instead of a full clear: every changed
+// segment is a preset's own (glob or vector on/off), never a global feature or embed change.
+export function isPresetOnlyChange(changedKeys) {
+    return changedKeys.size > 0 && [
+        ...changedKeys
+    ].every((key)=>key.startsWith('preset:'));
+}
+// One preset's old include/exclude/vectors-on-off, read back out of a stored `preset:<name>:...`
+// segment (config/access.ts's featureSignature format). Absent (a brand-new preset) is the empty
+// preset, not a parse failure; a segment present but not in that exact shape returns null so the
+// caller falls back to a full rebuild instead of guessing.
+function parseOldPreset(before, name) {
+    const part = before.split('|').find((p)=>p.startsWith(`preset:${name}:`));
+    if (part === undefined) return {
+        include: [],
+        exclude: [],
+        vectors: false
+    };
+    const fields = part.split(':');
+    if (fields.length !== 5) return null;
+    const [, , includeStr, excludeStr, onOff] = fields;
+    if (onOff !== 'on' && onOff !== 'off') return null;
+    return {
+        include: includeStr === '' ? [] : includeStr.split('+'),
+        exclude: excludeStr === '' ? [] : excludeStr.split('+'),
+        vectors: onOff === 'on'
+    };
+}
+// Paths one preset definition covers, via the real listFiles so stat filtering (directories,
+// dangling symlinks) matches production coverage exactly.
+function matchPreset(cfg, baseDir, name, def) {
+    const scoped = {
+        ...cfg,
+        presets: {
+            [name]: {
+                include: def.include,
+                exclude: def.exclude
+            }
+        }
+    };
+    return new Set(listFiles(scoped, baseDir).map((f)=>f.relPath));
+}
+// Every path whose coverage by a changed preset genuinely moved: gained or lost it (a glob edit),
+// or kept it under a flipped vectors on/off (that preset's whole match set, since its contribution
+// to the file's union embed flag may have changed even though membership itself did not). Returns
+// null when a changed preset's stored segment can't be parsed, so the caller takes the full rebuild.
+export function forcedPresetPaths(cfg, baseDir, before, changedKeys) {
+    const forced = new Set();
+    for (const key of changedKeys){
+        var _cfg_presets_name_exclude;
+        const name = key.slice('preset:'.length);
+        const oldDef = parseOldPreset(before, name);
+        if (oldDef === null) return null;
+        const stillDeclared = Object.hasOwn(cfg.presets, name);
+        const newDef = stillDeclared ? {
+            include: cfg.presets[name].include,
+            exclude: (_cfg_presets_name_exclude = cfg.presets[name].exclude) !== null && _cfg_presets_name_exclude !== void 0 ? _cfg_presets_name_exclude : [],
+            vectors: presetHasSignal(cfg, name, 'vectors')
+        } : {
+            include: [],
+            exclude: [],
+            vectors: false
+        };
+        const oldMatch = matchPreset(cfg, baseDir, name, oldDef);
+        const newMatch = stillDeclared ? matchPreset(cfg, baseDir, name, newDef) : new Set();
+        if (oldDef.vectors !== newDef.vectors) {
+            for (const p of oldMatch)forced.add(p);
+            for (const p of newMatch)forced.add(p);
+        } else {
+            for (const p of oldMatch)if (!newMatch.has(p)) forced.add(p);
+            for (const p of newMatch)if (!oldMatch.has(p)) forced.add(p);
+        }
+    }
+    return forced;
+}

package/dist/esm/store/preset-scope.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/preset-scope.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { presetHasSignal } from '../config/index.ts';\nimport { listFiles } from '../scan/index.ts';\n\n// A signature change open() can route to a narrow reparse instead of a full clear: every changed\n// segment is a preset's own (glob or vector on/off), never a global feature or embed change.\nexport function isPresetOnlyChange(changedKeys: ReadonlySet<string>): boolean {\n  return changedKeys.size > 0 && [...changedKeys].every((key) => key.startsWith('preset:'));\n}\n\ninterface PresetDef {\n  include: string[];\n  exclude: string[];\n  vectors: boolean;\n}\n\n// One preset's old include/exclude/vectors-on-off, read back out of a stored `preset:<name>:...`\n// segment (config/access.ts's featureSignature format). Absent (a brand-new preset) is the empty\n// preset, not a parse failure; a segment present but not in that exact shape returns null so the\n// caller falls back to a full rebuild instead of guessing.\nfunction parseOldPreset(before: string, name: string): PresetDef | null {\n  const part = before.split('|').find((p) => p.startsWith(`preset:${name}:`));\n  if (part === undefined) return { include: [], exclude: [], vectors: false };\n  const fields = part.split(':');\n  if (fields.length !== 5) return null;\n  const [, , includeStr, excludeStr, onOff] = fields;\n  if (onOff !== 'on' && onOff !== 'off') return null;\n  return { include: includeStr === '' ? [] : includeStr.split('+'), exclude: excludeStr === '' ? [] : excludeStr.split('+'), vectors: onOff === 'on' };\n}\n\n// Paths one preset definition covers, via the real listFiles so stat filtering (directories,\n// dangling symlinks) matches production coverage exactly.\nfunction matchPreset(cfg: Config, baseDir: string, name: string, def: PresetDef): Set<string> {\n  const scoped: Config = { ...cfg, presets: { [name]: { include: def.include, exclude: def.exclude } } };\n  return new Set(listFiles(scoped, baseDir).map((f) => f.relPath));\n}\n\n// Every path whose coverage by a changed preset genuinely moved: gained or lost it (a glob edit),\n// or kept it under a flipped vectors on/off (that preset's whole match set, since its contribution\n// to the file's union embed flag may have changed even though membership itself did not). Returns\n// null when a changed preset's stored segment can't be parsed, so the caller takes the full rebuild.\nexport function forcedPresetPaths(cfg: Config, baseDir: string, before: string, changedKeys: ReadonlySet<string>): Set<string> | null {\n  const forced = new Set<string>();\n  for (const key of changedKeys) {\n    const name = key.slice('preset:'.length);\n    const oldDef = parseOldPreset(before, name);\n    if (oldDef === null) return null;\n    const stillDeclared = Object.hasOwn(cfg.presets, name);\n    const newDef: PresetDef = stillDeclared ? { include: cfg.presets[name].include, exclude: cfg.presets[name].exclude ?? [], vectors: presetHasSignal(cfg, name, 'vectors') } : { include: [], exclude: [], vectors: false };\n\n    const oldMatch = matchPreset(cfg, baseDir, name, oldDef);\n    const newMatch = stillDeclared ? matchPreset(cfg, baseDir, name, newDef) : new Set<string>();\n\n    if (oldDef.vectors !== newDef.vectors) {\n      for (const p of oldMatch) forced.add(p);\n      for (const p of newMatch) forced.add(p);\n    } else {\n      for (const p of oldMatch) if (!newMatch.has(p)) forced.add(p);\n      for (const p of newMatch) if (!oldMatch.has(p)) forced.add(p);\n    }\n  }\n  return forced;\n}\n"],"names":["presetHasSignal","listFiles","isPresetOnlyChange","changedKeys","size","every","key","startsWith","parseOldPreset","before","name","part","split","find","p","undefined","include","exclude","vectors","fields","length","includeStr","excludeStr","onOff","matchPreset","cfg","baseDir","def","scoped","presets","Set","map","f","relPath","forcedPresetPaths","forced","slice","oldDef","stillDeclared","Object","hasOwn","newDef","oldMatch","newMatch","add","has"],"mappings":"AACA,SAASA,eAAe,QAAQ,qBAAqB;AACrD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,iGAAiG;AACjG,6FAA6F;AAC7F,OAAO,SAASC,mBAAmBC,WAAgC;IACjE,OAAOA,YAAYC,IAAI,GAAG,KAAK;WAAID;KAAY,CAACE,KAAK,CAAC,CAACC,MAAQA,IAAIC,UAAU,CAAC;AAChF;AAQA,iGAAiG;AACjG,iGAAiG;AACjG,iGAAiG;AACjG,2DAA2D;AAC3D,SAASC,eAAeC,MAAc,EAAEC,IAAY;IAClD,MAAMC,OAAOF,OAAOG,KAAK,CAAC,KAAKC,IAAI,CAAC,CAACC,IAAMA,EAAEP,UAAU,CAAC,CAAC,OAAO,EAAEG,KAAK,CAAC,CAAC;IACzE,IAAIC,SAASI,WAAW,OAAO;QAAEC,SAAS,EAAE;QAAEC,SAAS,EAAE;QAAEC,SAAS;IAAM;IAC1E,MAAMC,SAASR,KAAKC,KAAK,CAAC;IAC1B,IAAIO,OAAOC,MAAM,KAAK,GAAG,OAAO;IAChC,MAAM,KAAKC,YAAYC,YAAYC,MAAM,GAAGJ;IAC5C,IAAII,UAAU,QAAQA,UAAU,OAAO,OAAO;IAC9C,OAAO;QAAEP,SAASK,eAAe,KAAK,EAAE,GAAGA,WAAWT,KAAK,CAAC;QAAMK,SAASK,eAAe,KAAK,EAAE,GAAGA,WAAWV,KAAK,CAAC;QAAMM,SAASK,UAAU;IAAK;AACrJ;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,SAASC,YAAYC,GAAW,EAAEC,OAAe,EAAEhB,IAAY,EAAEiB,GAAc;IAC7E,MAAMC,SAAiB;QAAE,GAAGH,GAAG;QAAEI,SAAS;YAAE,CAACnB,KAAK,EAAE;gBAAEM,SAASW,IAAIX,OAAO;gBAAEC,SAASU,IAAIV,OAAO;YAAC;QAAE;IAAE;IACrG,OAAO,IAAIa,IAAI7B,UAAU2B,QAAQF,SAASK,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;AAChE;AAEA,kGAAkG;AAClG,mGAAmG;AACnG,kGAAkG;AAClG,qGAAqG;AACrG,OAAO,SAASC,kBAAkBT,GAAW,EAAEC,OAAe,EAAEjB,MAAc,EAAEN,WAAgC;IAC9G,MAAMgC,SAAS,IAAIL;IACnB,KAAK,MAAMxB,OAAOH,YAAa;YAK4DsB;QAJzF,MAAMf,OAAOJ,IAAI8B,KAAK,CAAC,UAAUhB,MAAM;QACvC,MAAMiB,SAAS7B,eAAeC,QAAQC;QACtC,IAAI2B,WAAW,MAAM,OAAO;QAC5B,MAAMC,gBAAgBC,OAAOC,MAAM,CAACf,IAAII,OAAO,EAAEnB;QACjD,MAAM+B,SAAoBH,gBAAgB;YAAEtB,SAASS,IAAII,OAAO,CAACnB,KAAK,CAACM,OAAO;YAAEC,OAAO,GAAEQ,4BAAAA,IAAII,OAAO,CAACnB,KAAK,CAACO,OAAO,cAAzBQ,uCAAAA,4BAA6B,EAAE;YAAEP,SAASlB,gBAAgByB,KAAKf,MAAM;QAAW,IAAI;YAAEM,SAAS,EAAE;YAAEC,SAAS,EAAE;YAAEC,SAAS;QAAM;QAExN,MAAMwB,WAAWlB,YAAYC,KAAKC,SAAShB,MAAM2B;QACjD,MAAMM,WAAWL,gBAAgBd,YAAYC,KAAKC,SAAShB,MAAM+B,UAAU,IAAIX;QAE/E,IAAIO,OAAOnB,OAAO,KAAKuB,OAAOvB,OAAO,EAAE;YACrC,KAAK,MAAMJ,KAAK4B,SAAUP,OAAOS,GAAG,CAAC9B;YACrC,KAAK,MAAMA,KAAK6B,SAAUR,OAAOS,GAAG,CAAC9B;QACvC,OAAO;YACL,KAAK,MAAMA,KAAK4B,SAAU,IAAI,CAACC,SAASE,GAAG,CAAC/B,IAAIqB,OAAOS,GAAG,CAAC9B;YAC3D,KAAK,MAAMA,KAAK6B,SAAU,IAAI,CAACD,SAASG,GAAG,CAAC/B,IAAIqB,OAAOS,GAAG,CAAC9B;QAC7D;IACF;IACA,OAAOqB;AACT"}

package/dist/esm/store/reconcile.d.ts

@@ -1,6 +1,7 @@
 import type { Config } from '../config/index.js';
+import type { ParsePool } from '../scan/pool.js';
 import type { Connection, ReconcileDialect } from './types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{
     parsed: number;
     warnings: string[];
 }>;

package/dist/esm/store/reconcile.js

@@ -18,7 +18,7 @@ const CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-export async function reconcile(conn, cfg, baseDir, dialect) {
+export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
     const files = listFiles(cfg, baseDir);
     const currentSet = new Set(files.map((f)=>f.relPath));
     const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
@@ -27,10 +27,16 @@ export async function reconcile(conn, cfg, baseDir, dialect) {
             r.path,
             r
         ]));
+    // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is
+    // already caught below by !currentSet.has, since it can only be forced by having existed under
+    // an old preset's match, which means it was reconciled into `existing` already.
     const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
+    // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its
+    // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.
     const toReparse = files.filter((f)=>{
+        var _ref;
         const row = existing.get(f.relPath);
-        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
+        return !row || row._mtime !== f.mtimeMs || row._size !== f.size || ((_ref = forcedPaths === null || forcedPaths === void 0 ? void 0 : forcedPaths.has(f.relPath)) !== null && _ref !== void 0 ? _ref : false);
     });
     if (vanished.length === 0 && toReparse.length === 0) return {
         parsed: 0,
@@ -41,7 +47,9 @@ export async function reconcile(conn, cfg, baseDir, dialect) {
     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
     // reconciles stay silent (progress() has a threshold).
     const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick, {
+        pool
+    });
     report.finish();
     for (const col of newColumns)seenColumns.add(col);
     const allColumns = [
@@ -77,7 +85,23 @@ export async function reconcile(conn, cfg, baseDir, dialect) {
         // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a
         // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.
         const present = await getColumns(conn);
-        for (const col of newColumns)if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}${dialect.columnTypeSuffix()}`);
+        const missingColumns = newColumns.filter((col)=>!present.has(col));
+        if (missingColumns.length > 0) await dialect.addColumns(conn, missingColumns);
+        // Revalidated once the lock is held, before this process's own frontmatter upsert below:
+        // `added` came from a path read taken before this transaction's lock, so a path still
+        // called "added" here may already have a content row from a concurrent reconcile that
+        // committed while this one waited. One SELECT, not a DELETE per added file -- with no
+        // contention it returns the same set `existing` already ruled out, so nothing extra clears.
+        let contentTouched = touched;
+        if (added.length > 0) {
+            const currentPathsStmt = await conn.prepare('SELECT "path" FROM frontmatter');
+            const currentPaths = new Set((await currentPathsStmt.all()).map((r)=>r.path));
+            const staleAdded = added.filter((p)=>currentPaths.has(p));
+            if (staleAdded.length > 0) contentTouched = [
+                ...touched,
+                ...staleAdded
+            ];
+        }
         if (parsedDocs.length > 0) {
             const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
                     var _doc_data_col;
@@ -93,7 +117,7 @@ export async function reconcile(conn, cfg, baseDir, dialect) {
         }
         // After the upsert, so every doc already has the frontmatter row sqlite's content rowid
         // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.
-        await dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg);
+        await dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg);
         // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
         // new docs have nothing to clear, which keeps cold builds linear.
         if (touched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map((p)=>[
@@ -104,7 +128,10 @@ export async function reconcile(conn, cfg, baseDir, dialect) {
             doc.relPath,
             presetName
         ]);
-        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
+        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken
+        // before this transaction's lock, so a path this process calls "added" can already have
+        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.
+        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows);
         // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's
         // node set, so a lingering vanished row would dilute rank mass across every surviving note.
         if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[

package/dist/esm/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      for (const col of newColumns) if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}${dialect.columnTypeSuffix()}`);\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg);\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      if (touched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          touched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      if (touched.length > 0) for (const feature of features) await feature.remove?.(conn, touched, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  if (dialect.recordDuration) await dialect.recordDuration(conn, durationMs);\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","files","currentSet","map","f","relPath","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","feature","present","exec","columnTypeSuffix","rows","doc","ctimeMs","parseError","data","runBatch","reconcileContent","presetRows","presetName","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAC/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACrD,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB;IACvG,MAAMC,QAAQb,UAAUU,KAAKC;IAC7B,MAAMG,aAAa,IAAIP,IAAIM,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaL,GAAG,CAAC,CAACS,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACV,WAAWc,GAAG,CAACJ,EAAEC,IAAI,GAAGV,GAAG,CAAC,CAACS,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYhB,MAAMc,MAAM,CAAC,CAACX;QAC9B,MAAMc,MAAMR,SAASS,GAAG,CAACf,EAAEC,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWzC,eAAeY;IAChC,MAAM8B,cAAc,MAAMrC,WAAWM;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAMgC,SAAS1C,SAAS,mBAAmB8B,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAM1C,aAAa2B,WAAWU,UAAU7B,KAAK8B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG5B,QAAQsC,gBAAgB,CAACD,WAAWb,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMe,kBAAkBF,WAAWtB,MAAM,CAAC,CAACyB,IAAM9C,yBAAyBsB,GAAG,CAACwB,MAAM,CAACnD,iBAAiB2B,GAAG,CAACwB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBpC,GAAG,CAACX,YAAYkD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBpC,GAAG,CAAC,IAAM,KAAKuC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLxB,MAAM,CAAC,CAACyB,IAAMA,MAAM,QACpBrC,GAAG,CAAC,CAACqC,IAAM,GAAGhD,WAAWgD,GAAG,YAAY,EAAEhD,WAAWgD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ1B,UAAUF,MAAM,CAAC,CAACX,IAAM,CAACM,SAASM,GAAG,CAACZ,EAAEC,OAAO,GAAGF,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACpF,MAAMuC,QAAwB;QAAE3C;QAAO4C,UAAUd,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO;QAAGsC;QAAO7B;IAAS;IACnG,MAAMiC,WAAW,IAAIpD,IAAIgD;IACzB,MAAMK,mBAAmBjB,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO,EAAEU,MAAM,CAAC,CAACkC,IAAM,CAACF,SAAS/B,GAAG,CAACiC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIpC;WAAakC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM5D,gBACJI,MACA;YA4CgEyD,iBAKxBA;QAhDtC,iGAAiG;QACjG,sFAAsF;QACtF,MAAMC,UAAU,MAAMhE,WAAWM;QACjC,KAAK,MAAMsC,OAAOH,WAAY,IAAI,CAACuB,QAAQvC,GAAG,CAACmB,MAAM,MAAMtC,KAAK2D,IAAI,CAAC,CAAC,mCAAmC,EAAEhE,WAAW2C,OAAOnC,QAAQyD,gBAAgB,IAAI;QAEzJ,IAAI1B,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMkC,OAAO3B,WAAW5B,GAAG,CAAC,CAACwD,MAC3BpB,gBAAgBpC,GAAG,CAAC,CAACgC;wBAOZwB;oBANP,IAAIxB,QAAQ,QAAQ,OAAOwB,IAAItD,OAAO;oBACtC,IAAI8B,QAAQ,UAAU,OAAOwB,IAAItC,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAOwB,IAAIC,OAAO;oBACxC,IAAIzB,QAAQ,SAAS,OAAOwB,IAAIpC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAOwB,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC3B,IAAI,cAAbwB,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAM9D,KAAKkE,QAAQ,CAACtB,WAAWiB;QACjC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAM1D,QAAQgE,gBAAgB,CAACnE,MAAMqD,SAASnB,YAAYa,OAAO9C;QAEjE,0FAA0F;QAC1F,kEAAkE;QAClE,IAAIoD,QAAQ1B,MAAM,GAAG,GACnB,MAAM3B,KAAKkE,QAAQ,CACjB,6CACAb,QAAQ/C,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAE1B,MAAMgB,aAA0B,EAAE;QAClC,KAAK,MAAMN,OAAO5B,WAAY,KAAK,MAAMmC,cAAcP,IAAIQ,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACT,IAAItD,OAAO;YAAE6D;SAAW;QAC7G,IAAID,WAAWzC,MAAM,GAAG,GAAG,MAAM3B,KAAKkE,QAAQ,CAAC,2DAA2DE;QAE1G,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAInD,SAASU,MAAM,GAAG,GACpB,MAAM3B,KAAKkE,QAAQ,CACjB,4CACAjD,SAASX,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAG3B,IAAIC,QAAQ1B,MAAM,GAAG,GAAG,KAAK,MAAM8B,WAAW3B,SAAU,QAAM2B,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBzD,MAAMqD,SAASN;QAC9F,KAAK,MAAMU,WAAW3B,SAAU;gBAExB2B;YADN,MAAMgB,iBAAiCvC,WAAW5B,GAAG,CAAC,CAACwD,MAAS,CAAA;oBAAE9C,MAAM8C,IAAItD,OAAO;oBAAEkE,WAAWZ,IAAIY,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBzD,MAAMyE,gBAAgB1B;QAC9C;QACA,KAAK,MAAMU,WAAW3B,SAAU,QAAM2B,0BAAAA,QAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,SAAyBzD,MAAM+C;IACvE,GACA5C,QAAQ2E,SAAS;IAGnB,MAAMC,aAAaxB,KAAKC,GAAG,KAAKF;IAChC,IAAInD,QAAQ6E,cAAc,EAAE,MAAM7E,QAAQ6E,cAAc,CAAChF,MAAM+E;IAE/D,OAAO;QAAEnD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool });\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await dialect.addColumns(conn, missingColumns);\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0) {\n        const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n        const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n        const staleAdded = added.filter((p) => currentPaths.has(p));\n        if (staleAdded.length > 0) contentTouched = [...touched, ...staleAdded];\n      }\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg);\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      if (touched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          touched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n      // before this transaction's lock, so a path this process calls \"added\" can already have\n      // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      if (touched.length > 0) for (const feature of features) await feature.remove?.(conn, touched, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  if (dialect.recordDuration) await dialect.recordDuration(conn, durationMs);\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","pool","forcedPaths","files","currentSet","map","f","relPath","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","feature","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","rows","doc","ctimeMs","parseError","data","runBatch","reconcileContent","presetRows","presetName","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACrD,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAC5J,MAAMC,QAAQf,UAAUU,KAAKC;IAC7B,MAAMK,aAAa,IAAIT,IAAIQ,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe,MAAMX,KAAKY,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaL,GAAG,CAAC,CAACS,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACV,WAAWc,GAAG,CAACJ,EAAEC,IAAI,GAAGV,GAAG,CAAC,CAACS,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYhB,MAAMc,MAAM,CAAC,CAACX;;QAC9B,MAAMc,MAAMR,SAASS,GAAG,CAACf,EAAEC,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI,aAAKvB,wBAAAA,kCAAAA,YAAagB,GAAG,CAACZ,EAAEC,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAW3C,eAAeY;IAChC,MAAMgC,cAAc,MAAMvC,WAAWM;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAMkC,SAAS5C,SAAS,mBAAmBgC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAM5C,aAAa6B,WAAWU,UAAU/B,KAAKgC,aAAaC,OAAOI,IAAI,EAAE;QAAElC;IAAK;IACjI8B,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG9B,QAAQwC,gBAAgB,CAACD,WAAWb,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMe,kBAAkBF,WAAWtB,MAAM,CAAC,CAACyB,IAAMhD,yBAAyBwB,GAAG,CAACwB,MAAM,CAACrD,iBAAiB6B,GAAG,CAACwB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBpC,GAAG,CAACb,YAAYoD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBpC,GAAG,CAAC,IAAM,KAAKuC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLxB,MAAM,CAAC,CAACyB,IAAMA,MAAM,QACpBrC,GAAG,CAAC,CAACqC,IAAM,GAAGlD,WAAWkD,GAAG,YAAY,EAAElD,WAAWkD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ1B,UAAUF,MAAM,CAAC,CAACX,IAAM,CAACM,SAASM,GAAG,CAACZ,EAAEC,OAAO,GAAGF,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACpF,MAAMuC,QAAwB;QAAE3C;QAAO4C,UAAUd,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO;QAAGsC;QAAO7B;IAAS;IACnG,MAAMiC,WAAW,IAAItD,IAAIkD;IACzB,MAAMK,mBAAmBjB,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO,EAAEU,MAAM,CAAC,CAACkC,IAAM,CAACF,SAAS/B,GAAG,CAACiC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIpC;WAAakC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM9D,gBACJI,MACA;YA6DgE2D,iBAKxBA;QAjEtC,iGAAiG;QACjG,sFAAsF;QACtF,MAAMC,UAAU,MAAMlE,WAAWM;QACjC,MAAM6D,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACoB,MAAQ,CAACoB,QAAQvC,GAAG,CAACmB;QAC/D,IAAIqB,eAAehC,MAAM,GAAG,GAAG,MAAM1B,QAAQ2D,UAAU,CAAC9D,MAAM6D;QAE9D,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBR;QACrB,IAAIP,MAAMnB,MAAM,GAAG,GAAG;YACpB,MAAMmC,mBAAmB,MAAMhE,KAAKY,OAAO,CAAC;YAC5C,MAAMqD,eAAe,IAAInE,IAAI,AAAE,CAAA,MAAMkE,iBAAiBlD,GAAG,EAAC,EAA+BN,GAAG,CAAC,CAACS,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAalB,MAAM5B,MAAM,CAAC,CAACkC,IAAMW,aAAa5C,GAAG,CAACiC;YACxD,IAAIY,WAAWrC,MAAM,GAAG,GAAGkC,iBAAiB;mBAAIR;mBAAYW;aAAW;QACzE;QAEA,IAAI9B,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,OAAO/B,WAAW5B,GAAG,CAAC,CAAC4D,MAC3BxB,gBAAgBpC,GAAG,CAAC,CAACgC;wBAOZ4B;oBANP,IAAI5B,QAAQ,QAAQ,OAAO4B,IAAI1D,OAAO;oBACtC,IAAI8B,QAAQ,UAAU,OAAO4B,IAAI1C,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO4B,IAAIC,OAAO;oBACxC,IAAI7B,QAAQ,SAAS,OAAO4B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO4B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC/B,IAAI,cAAb4B,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMpE,KAAKwE,QAAQ,CAAC1B,WAAWqB;QACjC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMhE,QAAQsE,gBAAgB,CAACzE,MAAM+D,gBAAgB3B,YAAYa,OAAOhD;QAExE,0FAA0F;QAC1F,kEAAkE;QAClE,IAAIsD,QAAQ1B,MAAM,GAAG,GACnB,MAAM7B,KAAKwE,QAAQ,CACjB,6CACAjB,QAAQ/C,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAE1B,MAAMoB,aAA0B,EAAE;QAClC,KAAK,MAAMN,OAAOhC,WAAY,KAAK,MAAMuC,cAAcP,IAAIQ,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACT,IAAI1D,OAAO;YAAEiE;SAAW;QAC7G,uFAAuF;QACvF,wFAAwF;QACxF,uGAAuG;QACvG,IAAID,WAAW7C,MAAM,GAAG,GAAG,MAAM7B,KAAKwE,QAAQ,CAAC,kGAAkGE;QAEjJ,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAIvD,SAASU,MAAM,GAAG,GACpB,MAAM7B,KAAKwE,QAAQ,CACjB,4CACArD,SAASX,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAG3B,IAAIC,QAAQ1B,MAAM,GAAG,GAAG,KAAK,MAAM8B,WAAW3B,SAAU,QAAM2B,kBAAAA,QAAQmB,MAAM,cAAdnB,sCAAAA,qBAAAA,SAAiB3D,MAAMuD,SAASN;QAC9F,KAAK,MAAMU,WAAW3B,SAAU;gBAExB2B;YADN,MAAMoB,iBAAiC3C,WAAW5B,GAAG,CAAC,CAAC4D,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAEsE,WAAWZ,IAAIY,SAAS,CAACrB,QAAQsB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMtB,iBAAAA,QAAQuB,KAAK,cAAbvB,qCAAAA,oBAAAA,SAAgB3D,MAAM+E,gBAAgB9B;QAC9C;QACA,KAAK,MAAMU,WAAW3B,SAAU,QAAM2B,0BAAAA,QAAQwB,cAAc,cAAtBxB,8CAAAA,6BAAAA,SAAyB3D,MAAMiD;IACvE,GACA9C,QAAQiF,SAAS;IAGnB,MAAMC,aAAa5B,KAAKC,GAAG,KAAKF;IAChC,IAAIrD,QAAQmF,cAAc,EAAE,MAAMnF,QAAQmF,cAAc,CAACtF,MAAMqF;IAE/D,OAAO;QAAEvD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/{sqlite/signature.d.ts → signature.d.ts}

@@ -1,6 +1,3 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
 export declare function changedSignatureKeys(before: string, after: string): Set<string>;
 export declare function embedIdentityAdopted(before: string, after: string): boolean;
 export declare function signatureDiff(before: string, after: string): string;
-export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;

package/dist/esm/store/signature.js

@@ -0,0 +1,37 @@
+// Feature-signature comparison for open()'s rebuild-vs-adopt decision (config.featureSignature's
+// format: one `feature:<name>:on|off` segment per feature, embed provider, then one segment per
+// preset). Pure string comparison, no SQL, shared by every store's open dialect.
+// Segment keys that moved between two feature signatures. `preset:` and `feature:` segments key
+// on their first two colon-separated parts (e.g. `preset:default`, `feature:tags`) so a change to
+// one preset or one feature is independently detectable; every other segment keys on its prefix alone.
+export function changedSignatureKeys(before, after) {
+    const keyOf = (part)=>part.startsWith('preset:') || part.startsWith('feature:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
+    const parse = (sig)=>new Map(sig.split('|').map((part)=>[
+                keyOf(part),
+                part
+            ]));
+    const a = parse(before);
+    const b = parse(after);
+    const changed = new Set();
+    for (const [key, val] of b)if (a.get(key) !== val) changed.add(key);
+    for (const key of a.keys())if (!b.has(key)) changed.add(key);
+    return changed;
+}
+// Whether the embed segment only gained its resolved weight identity: same provider and
+// model, no identity recorded before, one now -- adopted into meta without a rebuild.
+export function embedIdentityAdopted(before, after) {
+    const embedPart = (sig)=>sig.split('|').find((p)=>p.startsWith('embed:'));
+    const b = embedPart(before);
+    const a = embedPart(after);
+    if (b === undefined || a === undefined) return false;
+    const at = a.indexOf('@');
+    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
+}
+// Names what moved, for the rebuild notice.
+export function signatureDiff(before, after) {
+    const changed = changedSignatureKeys(before, after);
+    const label = (key)=>key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : key.startsWith('feature:') ? `feature "${key.slice(8)}"` : 'features';
+    return changed.size === 0 ? 'features' : [
+        ...changed
+    ].map(label).join(', ');
+}

package/dist/esm/store/signature.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/signature.ts"],"sourcesContent":["// Feature-signature comparison for open()'s rebuild-vs-adopt decision (config.featureSignature's\n// format: one `feature:<name>:on|off` segment per feature, embed provider, then one segment per\n// preset). Pure string comparison, no SQL, shared by every store's open dialect.\n\n// Segment keys that moved between two feature signatures. `preset:` and `feature:` segments key\n// on their first two colon-separated parts (e.g. `preset:default`, `feature:tags`) so a change to\n// one preset or one feature is independently detectable; every other segment keys on its prefix alone.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') || part.startsWith('feature:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : key.startsWith('feature:') ? `feature \"${key.slice(8)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n"],"names":["changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedIdentityAdopted","embedPart","find","p","undefined","at","indexOf","signatureDiff","label","size"],"mappings":"AAAA,iGAAiG;AACjG,gGAAgG;AAChG,iFAAiF;AAEjF,gGAAgG;AAChG,kGAAkG;AAClG,uGAAuG;AACvG,OAAO,SAASA,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,cAAcD,KAAKC,UAAU,CAAC,cAAcD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACvJ,MAAMG,QAAQ,CAACC,MAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,CAACR,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMS,IAAIJ,MAAMR;IAChB,MAAMa,IAAIL,MAAMP;IAChB,MAAMa,UAAU,IAAIC;IACpB,KAAK,MAAM,CAACC,KAAKC,IAAI,IAAIJ,EAAG,IAAID,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;IAChE,KAAK,MAAMA,OAAOJ,EAAEQ,IAAI,GAAI,IAAI,CAACP,EAAEQ,GAAG,CAACL,MAAMF,QAAQK,GAAG,CAACH;IACzD,OAAOF;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASQ,qBAAqBtB,MAAc,EAAEC,KAAa;IAChE,MAAMsB,YAAY,CAACd,MAAgBA,IAAIJ,KAAK,CAAC,KAAKmB,IAAI,CAAC,CAACC,IAAMA,EAAErB,UAAU,CAAC;IAC3E,MAAMS,IAAIU,UAAUvB;IACpB,MAAMY,IAAIW,UAAUtB;IACpB,IAAIY,MAAMa,aAAad,MAAMc,WAAW,OAAO;IAC/C,MAAMC,KAAKf,EAAEgB,OAAO,CAAC;IACrB,OAAOf,EAAEe,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKf,EAAEN,KAAK,CAAC,GAAGqB,QAAQd;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASgB,cAAc7B,MAAc,EAAEC,KAAa;IACzD,MAAMa,UAAUf,qBAAqBC,QAAQC;IAC7C,MAAM6B,QAAQ,CAACd,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAIZ,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAGU,IAAIZ,UAAU,CAAC,cAAc,CAAC,SAAS,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzL,OAAOQ,QAAQiB,IAAI,KAAK,IAAI,aAAa;WAAIjB;KAAQ,CAACH,GAAG,CAACmB,OAAOvB,IAAI,CAAC;AACxE"}

package/dist/esm/store/sqlite/lexical.d.ts

@@ -1,2 +1,2 @@
 import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
-export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, segmenting: boolean): Promise<LexicalHit[]>;
+export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;

package/dist/esm/store/sqlite/lexical.js

@@ -3,11 +3,11 @@ import { segmentMatch } from '../../text/segment.js';
 // (benchmark/reports/2026-08-23-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.
 const SNIPPET_BOUND = 16384;
 const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
-// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. `segmenting`
-// mirrors the predicate that populated the `_seg` sidecars (contentTokenize(cfg) === undefined): an unspaced-script run in `terms` becomes a quoted grapheme phrase against them only when they exist.
-export async function queryLexical(conn, terms, opts, segmenting) {
+// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. An
+// unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.
+export async function queryLexical(conn, terms, opts) {
     const { whereJoin, whereCond, scopeCond, limit } = opts;
-    const query = segmenting ? segmentMatch(terms) : terms;
+    const query = segmentMatch(terms);
     // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH
     // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.
     if (query.trim() === '') return [];

package/dist/esm/store/sqlite/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. `segmenting`\n// mirrors the predicate that populated the `_seg` sidecars (contentTokenize(cfg) === undefined): an unspaced-script run in `terms` becomes a quoted grapheme phrase against them only when they exist.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, segmenting: boolean): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmenting ? segmentMatch(terms) : terms;\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","SNIPPET_BOUND","WEIGHTED_BM25","queryLexical","conn","terms","opts","segmenting","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,iFAAiF;AACjF,+HAA+H;AAC/H,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,oGAAoG;AACpG,uMAAuM;AACvM,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB,EAAEC,UAAmB;IAChH,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAMM,QAAQL,aAAaP,aAAaK,SAASA;IACjD,2FAA2F;IAC3F,sGAAsG;IACtG,IAAIO,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,+DAA+D,EAAEb,cAAc,+EAA+E,EAAEO,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAER,cAAc,OAAO,EAAES,OAAO;IACjR,MAAMI,OAAO,MAAMX,KAAKY,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. An\n// unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmentMatch(terms);\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","SNIPPET_BOUND","WEIGHTED_BM25","queryLexical","conn","terms","opts","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,iFAAiF;AACjF,+HAA+H;AAC/H,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,0FAA0F;AAC1F,+FAA+F;AAC/F,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB;IAC3F,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGJ;IACnD,MAAMK,QAAQX,aAAaK;IAC3B,2FAA2F;IAC3F,sGAAsG;IACtG,IAAIM,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,+DAA+D,EAAEZ,cAAc,+EAA+E,EAAEM,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEP,cAAc,OAAO,EAAEQ,OAAO;IACjR,MAAMI,OAAO,MAAMV,KAAKW,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}