sensemaking 0.25.1 → 0.27.0

package/dist/esm/store/reconcile.js

@@ -20,18 +20,35 @@ export const CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-export async function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
-    const { rootDir, configDir } = paths;
+export function createReconcileTiming(cfg) {
     const start = process.hrtime.bigint();
-    const features = activeFeatures(cfg);
-    const stages = stageRecorder(features.map((f)=>f.name));
-    const elapsed = ()=>Number(process.hrtime.bigint() - start) / 1e6;
-    const files = await stages.time('list', ()=>listFiles(cfg, rootDir));
-    const currentSet = new Set(files.map((f)=>f.relPath));
-    const existingRows = await stages.time('existing', async ()=>{
+    return {
+        stages: stageRecorder(activeFeatures(cfg).map((feature)=>feature.name)),
+        elapsed: ()=>Number(process.hrtime.bigint() - start) / 1e6,
+        workerParseMs: 0,
+        txMs: 0
+    };
+}
+// Called inside the builder's short planning snapshot, alongside its metadata baseline. Parsing
+// happens only after that transaction closes; publication later rejects this state if generation
+// or configuration moved in the meantime.
+export async function readReconcileSnapshot(conn, timing) {
+    const existingRows = await timing.stages.time('existing', async ()=>{
         const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
         return await existingStmt.all();
     });
+    return {
+        existingRows,
+        seenColumns: await getColumns(conn)
+    };
+}
+export async function prepareReconcile(snapshot, cfg, paths, dialect, timing, pool, forcedPaths) {
+    const { rootDir, configDir } = paths;
+    const features = activeFeatures(cfg);
+    const { stages, elapsed } = timing;
+    const files = await stages.time('list', ()=>listFiles(cfg, rootDir));
+    const currentSet = new Set(files.map((f)=>f.relPath));
+    const existingRows = snapshot.existingRows;
     const existing = new Map(existingRows.map((r)=>[
             r.path,
             r
@@ -47,12 +64,17 @@ export async function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
         const row = existing.get(f.relPath);
         return !row || row._mtime !== f.mtimeMs || row._size !== f.size || ((_ref = forcedPaths === null || forcedPaths === void 0 ? void 0 : forcedPaths.has(f.relPath)) !== null && _ref !== void 0 ? _ref : false);
     });
-    if (vanished.length === 0 && toReparse.length === 0) return {
-        parsed: 0,
-        warnings: [],
-        stages: stages.take(elapsed(), 0)
-    };
-    const seenColumns = await getColumns(conn);
+    if (vanished.length === 0 && toReparse.length === 0) {
+        return {
+            mutated: false,
+            parsed: 0,
+            warnings: [],
+            apply: async ()=>{},
+            finish: (txMs)=>stages.take(elapsed(), txMs, timing.workerParseMs),
+            record: async ()=>{}
+        };
+    }
+    const seenColumns = new Set(snapshot.seenColumns);
     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
     // reconciles stay silent (progress() has a threshold).
     const report = progress('reparsing files', toReparse.length);
@@ -60,6 +82,7 @@ export async function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
     const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', ()=>reparseFiles(toReparse, features, cfg, seenColumns, report.tick, {
             pool
         }));
+    timing.workerParseMs += workerParseMs;
     report.finish();
     for (const col of newColumns)seenColumns.add(col);
     const allColumns = [
@@ -89,118 +112,117 @@ export async function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
         ...vanished,
         ...reparsedExisting
     ];
-    const txStart = Date.now();
-    await withTransaction(conn, async ()=>{
-        // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a
-        // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.
-        const present = await getColumns(conn);
-        const missingColumns = newColumns.filter((col)=>!present.has(col));
-        if (missingColumns.length > 0) await stages.time('alter', ()=>dialect.addColumns(conn, missingColumns));
-        // Revalidated once the lock is held, before this process's own frontmatter upsert below:
-        // `added` came from a path read taken before this transaction's lock, so a path still
-        // called "added" here may already have a content row from a concurrent reconcile that
-        // committed while this one waited. One SELECT, not a DELETE per added file -- with no
-        // contention it returns the same set `existing` already ruled out, so nothing extra clears.
-        let contentTouched = touched;
-        if (added.length > 0) contentTouched = await stages.time('added-recheck', async ()=>{
-            const currentPathsStmt = await conn.prepare('SELECT "path" FROM frontmatter');
-            const currentPaths = new Set((await currentPathsStmt.all()).map((r)=>r.path));
-            const staleAdded = added.filter((p)=>currentPaths.has(p));
-            return staleAdded.length > 0 ? [
-                ...touched,
-                ...staleAdded
-            ] : touched;
-        });
-        if (parsedDocs.length > 0) {
-            const toRow = (doc)=>writableColumns.map((col)=>{
-                    var _doc_data_col;
-                    if (col === 'path') return doc.relPath;
-                    if (col === '_mtime') return doc.mtimeMs;
-                    if (col === '_ctime') return doc.ctimeMs;
-                    if (col === '_size') return doc.size;
-                    // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
-                    if (col === '_parse_error') return doc.parseError;
-                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
+    return {
+        mutated: true,
+        parsed: parsedDocs.length,
+        warnings,
+        async apply (conn) {
+            // The generation fence already proved the planning snapshot current under this write lock.
+            // Re-read columns only because schema setup is not itself a core generation publication.
+            const present = await getColumns(conn);
+            const missingColumns = newColumns.filter((col)=>!present.has(col));
+            if (missingColumns.length > 0) await stages.time('alter', ()=>dialect.addColumns(conn, missingColumns));
+            if (parsedDocs.length > 0) {
+                const toRow = (doc)=>writableColumns.map((col)=>{
+                        var _doc_data_col;
+                        if (col === 'path') return doc.relPath;
+                        if (col === '_mtime') return doc.mtimeMs;
+                        if (col === '_ctime') return doc.ctimeMs;
+                        if (col === '_size') return doc.size;
+                        if (col === '_parse_error') return doc.parseError;
+                        return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
+                    });
+                await stages.time('fm-upsert', async ()=>{
+                    const newDocs = parsedDocs.filter((d)=>addedSet.has(d.relPath));
+                    const updateDocs = parsedDocs.filter((d)=>!addedSet.has(d.relPath));
+                    await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));
+                    if (updateDocs.length > 0) {
+                        const rows = updateDocs.map(toRow);
+                        if (dialect.updateFrontmatter) await dialect.updateFrontmatter(conn, writableColumns, rows);
+                        else await conn.runBatch(insertSql, rows);
+                    }
                 });
-            // A path in `added` has no existing frontmatter row, so it can never conflict; the rest
-            // genuinely can, and keep the upsert.
-            await stages.time('fm-upsert', async ()=>{
-                const newDocs = parsedDocs.filter((d)=>addedSet.has(d.relPath));
-                const updateDocs = parsedDocs.filter((d)=>!addedSet.has(d.relPath));
-                await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));
-                if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));
-            });
-        }
-        // After the upsert, so every doc already has the frontmatter row sqlite's content rowid
-        // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.
-        await stages.time('text-index', ()=>dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));
-        // Exact source text belongs to the same indexed generation as content and section ranges.
-        // The upsert also covers an added path another writer committed while this build waited.
-        if (contentTouched.length > 0) await conn.runBatch('DELETE FROM indexed_sources WHERE "path" = ?', contentTouched.map((p)=>[
-                p
-            ]));
-        await appendRows(conn, 'indexed_sources', [
-            'path',
-            'text'
-        ], 'INSERT INTO indexed_sources ("path", text) VALUES (?, ?) ON CONFLICT("path") DO UPDATE SET text = excluded.text', parsedDocs.map((doc)=>[
-                doc.relPath,
-                doc.source
-            ]));
-        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
-        // new docs have nothing to clear, which keeps cold builds linear.
-        await stages.time('presets', async ()=>{
-            if (touched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map((p)=>[
+            }
+            await stages.time('text-index', ()=>dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg));
+            if (touched.length > 0) await conn.runBatch('DELETE FROM indexed_sources WHERE "path" = ?', touched.map((p)=>[
                     p
                 ]));
-            const presetRows = [];
-            for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
-                doc.relPath,
-                presetName
-            ]);
-            // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken
-            // before this transaction's lock, so a path this process calls "added" can already have
-            // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.
-            await appendRows(conn, 'preset_files', [
+            await appendRows(conn, 'indexed_sources', [
                 'path',
-                'preset'
-            ], 'INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows);
-        });
-        // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's
-        // node set, so a lingering vanished row would dilute rank mass across every surviving note.
-        if (vanished.length > 0) await stages.time('vanished', ()=>conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
-                    p
-                ])));
-        // Timed per feature per hook, so link resolution and PageRank are named stages without
-        // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.
-        if (touched.length > 0) for (const feature of features)await stages.time(featureStage(feature.name, 'remove'), ()=>{
-            var _feature_remove;
-            return (_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta);
-        });
-        for (const feature of features){
-            const docsForFeature = parsedDocs.map((doc)=>({
-                    path: doc.relPath,
-                    extracted: doc.extracted[feature.name]
-                }));
-            await stages.time(featureStage(feature.name, 'store'), ()=>{
-                var _feature_store;
-                return (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta);
+                'text'
+            ], 'INSERT INTO indexed_sources ("path", text) VALUES (?, ?) ON CONFLICT("path") DO UPDATE SET text = excluded.text', parsedDocs.map((doc)=>[
+                    doc.relPath,
+                    doc.source
+                ]));
+            await stages.time('presets', async ()=>{
+                if (touched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map((p)=>[
+                        p
+                    ]));
+                const presetRows = [];
+                for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
+                    doc.relPath,
+                    presetName
+                ]);
+                await appendRows(conn, 'preset_files', [
+                    'path',
+                    'preset'
+                ], 'INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows);
+            });
+            if (vanished.length > 0) await stages.time('vanished', ()=>conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
+                        p
+                    ])));
+            if (touched.length > 0) for (const feature of features)await stages.time(featureStage(feature.name, 'remove'), ()=>{
+                var _feature_remove;
+                return (_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta);
+            });
+            for (const feature of features){
+                const docsForFeature = parsedDocs.map((doc)=>({
+                        path: doc.relPath,
+                        extracted: doc.extracted[feature.name]
+                    }));
+                await stages.time(featureStage(feature.name, 'store'), ()=>{
+                    var _feature_store;
+                    return (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta);
+                });
+            }
+            for (const feature of features)await stages.time(featureStage(feature.name, 'after'), ()=>{
+                var _feature_afterReconcile;
+                return (_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta);
+            });
+        },
+        finish: (txMs)=>stages.take(elapsed(), txMs, timing.workerParseMs),
+        async record (conn, txMs) {
+            recordLockWaitMs(configDir, txMs);
+            if (dialect.recordDuration) await stages.time('meta', ()=>{
+                var _dialect_recordDuration;
+                return (_dialect_recordDuration = dialect.recordDuration) === null || _dialect_recordDuration === void 0 ? void 0 : _dialect_recordDuration.call(dialect, conn, txMs);
             });
         }
-        for (const feature of features)await stages.time(featureStage(feature.name, 'after'), ()=>{
-            var _feature_afterReconcile;
-            return (_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta);
-        });
-    }, dialect.beginMode());
-    const durationMs = Date.now() - txStart;
-    // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.
-    recordLockWaitMs(configDir, durationMs);
-    if (dialect.recordDuration) await stages.time('meta', ()=>{
-        var _dialect_recordDuration;
-        return (_dialect_recordDuration = dialect.recordDuration) === null || _dialect_recordDuration === void 0 ? void 0 : _dialect_recordDuration.call(dialect, conn, durationMs);
-    });
+    };
+}
+// Lower-level entrypoint retained only for focused reconcile tests. It has no generation fence;
+// production publication must go through builder.ts, and any new caller must provide exclusivity.
+export async function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
+    const timing = createReconcileTiming(cfg);
+    const snapshot = await withTransaction(conn, ()=>readReconcileSnapshot(conn, timing));
+    const plan = await prepareReconcile(snapshot, cfg, paths, dialect, timing, pool, forcedPaths);
+    if (!plan.mutated) return {
+        parsed: plan.parsed,
+        warnings: plan.warnings,
+        stages: plan.finish(0)
+    };
+    const txStart = Date.now();
+    let txMs;
+    try {
+        await withTransaction(conn, ()=>plan.apply(conn), dialect.beginMode());
+    } finally{
+        txMs = Date.now() - txStart;
+        timing.txMs += txMs;
+    }
+    await plan.record(conn, txMs);
     return {
-        parsed: parsedDocs.length,
-        warnings,
-        stages: stages.take(elapsed(), durationMs, workerParseMs)
+        parsed: plan.parsed,
+        warnings: plan.warnings,
+        stages: plan.finish(timing.txMs)
     };
 }

package/dist/esm/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport interface ReconcilePaths {\n  rootDir: string;\n  configDir: string;\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const { rootDir, configDir } = paths;\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, rootDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // Exact source text belongs to the same indexed generation as content and section ranges.\n      // The upsert also covers an added path another writer committed while this build waited.\n      if (contentTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM indexed_sources WHERE \"path\" = ?',\n          contentTouched.map((p) => [p])\n        );\n      await appendRows(\n        conn,\n        'indexed_sources',\n        ['path', 'text'],\n        'INSERT INTO indexed_sources (\"path\", text) VALUES (?, ?) ON CONFLICT(\"path\") DO UPDATE SET text = excluded.text',\n        parsedDocs.map((doc) => [doc.relPath, doc.source])\n      );\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(configDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","appendRows","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","paths","dialect","pool","forcedPaths","rootDir","configDir","start","process","hrtime","bigint","features","stages","map","f","name","elapsed","Number","files","time","currentSet","relPath","existingRows","existingStmt","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","take","seenColumns","report","docs","parsedDocs","newColumns","workerParseMs","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","toRow","doc","ctimeMs","parseError","data","newDocs","updateDocs","runBatch","reconcileContent","source","presetRows","presetName","presets","push","feature","remove","docsForFeature","extracted","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACjE,SAASC,YAAY,EAAeC,aAAa,QAAQ,cAAc;AACvE,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AAOvG,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,KAAqB,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAClK,MAAM,EAAEC,OAAO,EAAEC,SAAS,EAAE,GAAGL;IAC/B,MAAMM,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,MAAMC,WAAW3B,eAAegB;IAChC,MAAMY,SAASlB,cAAciB,SAASE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvD,MAAMC,UAAU,IAAMC,OAAOT,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;IAChE,MAAMW,QAAQ,MAAMN,OAAOO,IAAI,CAAC,QAAQ,IAAMjC,UAAUc,KAAKK;IAC7D,MAAMe,aAAa,IAAIvB,IAAIqB,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IAErD,MAAMC,eAAe,MAAMV,OAAOO,IAAI,CAAC,YAAY;QACjD,MAAMI,eAAe,MAAMxB,KAAKyB,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,MAAMC,WAAW,IAAIC,IAAIL,aAAaT,GAAG,CAAC,CAACe,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWR,aAAaS,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGhB,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACjB;;QAC9B,MAAMoB,MAAMR,SAASS,GAAG,CAACrB,EAAEO,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKtB,EAAEuB,OAAO,IAAIH,IAAII,KAAK,KAAKxB,EAAEyB,IAAI,aAAKnC,wBAAAA,kCAAAA,YAAa4B,GAAG,CAAClB,EAAEO,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;QAAE9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAW;IAAG;IAEzH,MAAM4B,cAAc,MAAMrD,WAAWQ;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8C,SAAS5D,SAAS,mBAAmBgD,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAEC,aAAa,EAAE,GAAG,MAAMrC,OAAOO,IAAI,CAAC,SAAS,IAAM/B,aAAa6C,WAAWtB,UAAUX,KAAK4C,aAAaC,OAAOK,IAAI,EAAE;YAAE/C;QAAK;IAC3K0C,OAAOM,MAAM;IACb,KAAK,MAAMC,OAAOJ,WAAYJ,YAAYS,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIV;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG1C,QAAQqD,gBAAgB,CAACD,WAAWd,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMgB,kBAAkBF,WAAWvB,MAAM,CAAC,CAAC0B,IAAM7D,yBAAyBoC,GAAG,CAACyB,MAAM,CAACtE,iBAAiB6C,GAAG,CAACyB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB3C,GAAG,CAACrB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB3C,GAAG,CAAC,IAAM,KAAK8C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLzB,MAAM,CAAC,CAAC0B,IAAMA,MAAM,QACpB5C,GAAG,CAAC,CAAC4C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ3B,UAAUF,MAAM,CAAC,CAACjB,IAAM,CAACY,SAASM,GAAG,CAAClB,EAAEO,OAAO,GAAGR,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IACpF,MAAMwC,QAAwB;QAAE3C;QAAO4C,UAAUf,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO;QAAGuC;QAAO9B;IAAS;IACnG,MAAMkC,WAAW,IAAInE,IAAI+D;IACzB,MAAMK,mBAAmBlB,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO,EAAEU,MAAM,CAAC,CAACmC,IAAM,CAACF,SAAShC,GAAG,CAACkC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIrC;WAAamC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM3E,gBACJI,MACA;QACE,iGAAiG;QACjG,sFAAsF;QACtF,MAAMwE,UAAU,MAAMhF,WAAWQ;QACjC,MAAMyE,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACqB,MAAQ,CAACmB,QAAQvC,GAAG,CAACoB;QAC/D,IAAIoB,eAAehC,MAAM,GAAG,GAAG,MAAM5B,OAAOO,IAAI,CAAC,SAAS,IAAMjB,QAAQuE,UAAU,CAAC1E,MAAMyE;QAEzF,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBP;QACrB,IAAIP,MAAMpB,MAAM,GAAG,GACjBkC,iBAAiB,MAAM9D,OAAOO,IAAI,CAAC,iBAAiB;YAClD,MAAMwD,mBAAmB,MAAM5E,KAAKyB,OAAO,CAAC;YAC5C,MAAMoD,eAAe,IAAI/E,IAAI,AAAE,CAAA,MAAM8E,iBAAiBlD,GAAG,EAAC,EAA+BZ,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAajB,MAAM7B,MAAM,CAAC,CAACmC,IAAMU,aAAa5C,GAAG,CAACkC;YACxD,OAAOW,WAAWrC,MAAM,GAAG,IAAI;mBAAI2B;mBAAYU;aAAW,GAAGV;QAC/D;QAEF,IAAIpB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,QAAQ,CAACC,MACbvB,gBAAgB3C,GAAG,CAAC,CAACuC;wBAOZ2B;oBANP,IAAI3B,QAAQ,QAAQ,OAAO2B,IAAI1D,OAAO;oBACtC,IAAI+B,QAAQ,UAAU,OAAO2B,IAAI1C,OAAO;oBACxC,IAAIe,QAAQ,UAAU,OAAO2B,IAAIC,OAAO;oBACxC,IAAI5B,QAAQ,SAAS,OAAO2B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIa,QAAQ,gBAAgB,OAAO2B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC9B,IAAI,cAAb2B,2BAAAA,gBAAiB;gBAC1B;YACF,wFAAwF;YACxF,sCAAsC;YACtC,MAAMnE,OAAOO,IAAI,CAAC,aAAa;gBAC7B,MAAMgE,UAAUpC,WAAWhB,MAAM,CAAC,CAACgC,IAAMC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBAC/D,MAAM+D,aAAarC,WAAWhB,MAAM,CAAC,CAACgC,IAAM,CAACC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBACnE,MAAM/B,WAAWS,MAAM,eAAeyD,iBAAiBE,WAAWyB,QAAQtE,GAAG,CAACiE;gBAC9E,IAAIM,WAAW5C,MAAM,GAAG,GAAG,MAAMzC,KAAKsF,QAAQ,CAAC3B,WAAW0B,WAAWvE,GAAG,CAACiE;YAC3E;QACF;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMlE,OAAOO,IAAI,CAAC,cAAc,IAAMjB,QAAQoF,gBAAgB,CAACvF,MAAM2E,gBAAgB3B,YAAYc,OAAO7D;QAExG,0FAA0F;QAC1F,yFAAyF;QACzF,IAAI0E,eAAelC,MAAM,GAAG,GAC1B,MAAMzC,KAAKsF,QAAQ,CACjB,gDACAX,eAAe7D,GAAG,CAAC,CAACqD,IAAM;gBAACA;aAAE;QAEjC,MAAM5E,WACJS,MACA,mBACA;YAAC;YAAQ;SAAO,EAChB,mHACAgD,WAAWlC,GAAG,CAAC,CAACkE,MAAQ;gBAACA,IAAI1D,OAAO;gBAAE0D,IAAIQ,MAAM;aAAC;QAGnD,0FAA0F;QAC1F,kEAAkE;QAClE,MAAM3E,OAAOO,IAAI,CAAC,WAAW;YAC3B,IAAIgD,QAAQ3B,MAAM,GAAG,GACnB,MAAMzC,KAAKsF,QAAQ,CACjB,6CACAlB,QAAQtD,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;YAE1B,MAAMsB,aAA0B,EAAE;YAClC,KAAK,MAAMT,OAAOhC,WAAY,KAAK,MAAM0C,cAAcV,IAAIW,OAAO,CAAEF,WAAWG,IAAI,CAAC;gBAACZ,IAAI1D,OAAO;gBAAEoE;aAAW;YAC7G,uFAAuF;YACvF,wFAAwF;YACxF,uGAAuG;YACvG,MAAMnG,WAAWS,MAAM,gBAAgB;gBAAC;gBAAQ;aAAS,EAAE,kGAAkGyF;QAC/J;QAEA,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAI1D,SAASU,MAAM,GAAG,GACpB,MAAM5B,OAAOO,IAAI,CAAC,YAAY,IAC5BpB,KAAKsF,QAAQ,CACX,4CACAvD,SAASjB,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;QAI7B,uFAAuF;QACvF,0FAA0F;QAC1F,IAAIC,QAAQ3B,MAAM,GAAG,GAAG,KAAK,MAAMoD,WAAWjF,SAAU,MAAMC,OAAOO,IAAI,CAAC1B,aAAamG,QAAQ7E,IAAI,EAAE,WAAW;gBAAM6E;oBAAAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,qBAAAA,SAAiB7F,MAAMoE,SAASN;;QACtJ,KAAK,MAAM+B,WAAWjF,SAAU;YAC9B,MAAMmF,iBAAiC/C,WAAWlC,GAAG,CAAC,CAACkE,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAE0E,WAAWhB,IAAIgB,SAAS,CAACH,QAAQ7E,IAAI,CAAC;gBAAC,CAAA;YAC5H,MAAMH,OAAOO,IAAI,CAAC1B,aAAamG,QAAQ7E,IAAI,EAAE,UAAU;oBAAM6E;wBAAAA,iBAAAA,QAAQI,KAAK,cAAbJ,qCAAAA,oBAAAA,SAAgB7F,MAAM+F,gBAAgBjC;;QACrG;QACA,KAAK,MAAM+B,WAAWjF,SAAU,MAAMC,OAAOO,IAAI,CAAC1B,aAAamG,QAAQ7E,IAAI,EAAE,UAAU;gBAAM6E;oBAAAA,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyB7F,MAAM8D;;IAC9H,GACA3D,QAAQgG,SAAS;IAGnB,MAAMC,aAAa9B,KAAKC,GAAG,KAAKF;IAChC,2GAA2G;IAC3G/E,iBAAiBiB,WAAW6F;IAC5B,IAAIjG,QAAQkG,cAAc,EAAE,MAAMxF,OAAOO,IAAI,CAAC,QAAQ;YAAMjB;gBAAAA,0BAAAA,QAAQkG,cAAc,cAAtBlG,8CAAAA,6BAAAA,SAAyBH,MAAMoG;;IAE3F,OAAO;QAAE1D,QAAQM,WAAWP,MAAM;QAAEE;QAAU9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAWmF,YAAYlD;IAAe;AAC1G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type StageRecorder, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport interface ReconcilePaths {\n  rootDir: string;\n  configDir: string;\n}\n\nexport interface ReconcileSnapshot {\n  existingRows: Array<{ path: string; _mtime: number; _size: number }>;\n  seenColumns: Set<string>;\n}\n\nexport interface ReconcilePlan {\n  mutated: boolean;\n  parsed: number;\n  warnings: string[];\n  apply(conn: Connection): Promise<void>;\n  finish(txMs: number): Stages;\n  record(conn: Connection, txMs: number): Promise<void>;\n}\n\nexport interface ReconcileTiming {\n  stages: StageRecorder;\n  elapsed(): number;\n  workerParseMs: number;\n  txMs: number;\n}\n\nexport function createReconcileTiming(cfg: Config): ReconcileTiming {\n  const start = process.hrtime.bigint();\n  return {\n    stages: stageRecorder(activeFeatures(cfg).map((feature) => feature.name)),\n    elapsed: () => Number(process.hrtime.bigint() - start) / 1e6,\n    workerParseMs: 0,\n    txMs: 0,\n  };\n}\n\n// Called inside the builder's short planning snapshot, alongside its metadata baseline. Parsing\n// happens only after that transaction closes; publication later rejects this state if generation\n// or configuration moved in the meantime.\nexport async function readReconcileSnapshot(conn: Connection, timing: ReconcileTiming): Promise<ReconcileSnapshot> {\n  const existingRows = await timing.stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as ReconcileSnapshot['existingRows'];\n  });\n  return { existingRows, seenColumns: await getColumns(conn) };\n}\n\nexport async function prepareReconcile(snapshot: ReconcileSnapshot, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, timing: ReconcileTiming, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<ReconcilePlan> {\n  const { rootDir, configDir } = paths;\n  const features = activeFeatures(cfg);\n  const { stages, elapsed } = timing;\n  const files = await stages.time('list', () => listFiles(cfg, rootDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n  const existingRows = snapshot.existingRows;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) {\n    return {\n      mutated: false,\n      parsed: 0,\n      warnings: [],\n      apply: async () => {},\n      finish: (txMs) => stages.take(elapsed(), txMs, timing.workerParseMs),\n      record: async () => {},\n    };\n  }\n\n  const seenColumns = new Set(snapshot.seenColumns);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  timing.workerParseMs += workerParseMs;\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  return {\n    mutated: true,\n    parsed: parsedDocs.length,\n    warnings,\n    async apply(conn) {\n      // The generation fence already proved the planning snapshot current under this write lock.\n      // Re-read columns only because schema setup is not itself a core generation publication.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) {\n            const rows = updateDocs.map(toRow);\n            if (dialect.updateFrontmatter) await dialect.updateFrontmatter(conn, writableColumns, rows);\n            else await conn.runBatch(insertSql, rows);\n          }\n        });\n      }\n\n      await stages.time('text-index', () => dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg));\n      if (touched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM indexed_sources WHERE \"path\" = ?',\n          touched.map((p) => [p])\n        );\n      await appendRows(\n        conn,\n        'indexed_sources',\n        ['path', 'text'],\n        'INSERT INTO indexed_sources (\"path\", text) VALUES (?, ?) ON CONFLICT(\"path\") DO UPDATE SET text = excluded.text',\n        parsedDocs.map((doc) => [doc.relPath, doc.source])\n      );\n\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    finish: (txMs) => stages.take(elapsed(), txMs, timing.workerParseMs),\n    async record(conn, txMs) {\n      recordLockWaitMs(configDir, txMs);\n      if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, txMs));\n    },\n  };\n}\n\n// Lower-level entrypoint retained only for focused reconcile tests. It has no generation fence;\n// production publication must go through builder.ts, and any new caller must provide exclusivity.\nexport async function reconcile(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const timing = createReconcileTiming(cfg);\n  const snapshot = await withTransaction(conn, () => readReconcileSnapshot(conn, timing));\n  const plan = await prepareReconcile(snapshot, cfg, paths, dialect, timing, pool, forcedPaths);\n  if (!plan.mutated) return { parsed: plan.parsed, warnings: plan.warnings, stages: plan.finish(0) };\n  const txStart = Date.now();\n  let txMs: number;\n  try {\n    await withTransaction(conn, () => plan.apply(conn), dialect.beginMode());\n  } finally {\n    txMs = Date.now() - txStart;\n    timing.txMs += txMs;\n  }\n  await plan.record(conn, txMs);\n  return { parsed: plan.parsed, warnings: plan.warnings, stages: plan.finish(timing.txMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","appendRows","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","createReconcileTiming","cfg","start","process","hrtime","bigint","stages","map","feature","name","elapsed","Number","workerParseMs","txMs","readReconcileSnapshot","conn","timing","existingRows","time","existingStmt","prepare","all","seenColumns","prepareReconcile","snapshot","paths","dialect","pool","forcedPaths","rootDir","configDir","features","files","currentSet","f","relPath","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","mutated","parsed","warnings","apply","finish","take","record","report","docs","parsedDocs","newColumns","tick","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","present","missingColumns","addColumns","toRow","doc","ctimeMs","parseError","data","newDocs","updateDocs","rows","updateFrontmatter","runBatch","reconcileContent","source","presetRows","presetName","presets","push","remove","docsForFeature","extracted","store","afterReconcile","recordDuration","reconcile","plan","txStart","Date","now","beginMode"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACjE,SAASC,YAAY,EAAmCC,aAAa,QAAQ,cAAc;AAC3F,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AA4BvG,OAAO,SAASC,sBAAsBC,GAAW;IAC/C,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,OAAO;QACLC,QAAQV,cAAcV,eAAee,KAAKM,GAAG,CAAC,CAACC,UAAYA,QAAQC,IAAI;QACvEC,SAAS,IAAMC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QACzDU,eAAe;QACfC,MAAM;IACR;AACF;AAEA,gGAAgG;AAChG,iGAAiG;AACjG,0CAA0C;AAC1C,OAAO,eAAeC,sBAAsBC,IAAgB,EAAEC,MAAuB;IACnF,MAAMC,eAAe,MAAMD,OAAOV,MAAM,CAACY,IAAI,CAAC,YAAY;QACxD,MAAMC,eAAe,MAAMJ,KAAKK,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,OAAO;QAAEJ;QAAcK,aAAa,MAAM7B,WAAWsB;IAAM;AAC7D;AAEA,OAAO,eAAeQ,iBAAiBC,QAA2B,EAAEvB,GAAW,EAAEwB,KAAqB,EAAEC,OAAyB,EAAEV,MAAuB,EAAEW,IAAgB,EAAEC,WAAiC;IAC7M,MAAM,EAAEC,OAAO,EAAEC,SAAS,EAAE,GAAGL;IAC/B,MAAMM,WAAW7C,eAAee;IAChC,MAAM,EAAEK,MAAM,EAAEI,OAAO,EAAE,GAAGM;IAC5B,MAAMgB,QAAQ,MAAM1B,OAAOY,IAAI,CAAC,QAAQ,IAAM9B,UAAUa,KAAK4B;IAC7D,MAAMI,aAAa,IAAIlC,IAAIiC,MAAMzB,GAAG,CAAC,CAAC2B,IAAMA,EAAEC,OAAO;IACrD,MAAMlB,eAAeO,SAASP,YAAY;IAC1C,MAAMmB,WAAW,IAAIC,IAAIpB,aAAaV,GAAG,CAAC,CAAC+B,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWvB,aAAawB,MAAM,CAAC,CAACH,IAAM,CAACL,WAAWS,GAAG,CAACJ,EAAEC,IAAI,GAAGhC,GAAG,CAAC,CAAC+B,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;;QAC9B,MAAMU,MAAMR,SAASS,GAAG,CAACX,EAAEC,OAAO;QAClC,OAAO,CAACS,OAAOA,IAAIE,MAAM,KAAKZ,EAAEa,OAAO,IAAIH,IAAII,KAAK,KAAKd,EAAEe,IAAI,aAAKrB,wBAAAA,kCAAAA,YAAac,GAAG,CAACR,EAAEC,OAAO,wCAAK;IACrG;IAEA,IAAIK,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG;QACnD,OAAO;YACLC,SAAS;YACTC,QAAQ;YACRC,UAAU,EAAE;YACZC,OAAO,WAAa;YACpBC,QAAQ,CAAC1C,OAASP,OAAOkD,IAAI,CAAC9C,WAAWG,MAAMG,OAAOJ,aAAa;YACnE6C,QAAQ,WAAa;QACvB;IACF;IAEA,MAAMnC,cAAc,IAAIvB,IAAIyB,SAASF,WAAW;IAEhD,oFAAoF;IACpF,uDAAuD;IACvD,MAAMoC,SAASvE,SAAS,mBAAmBwD,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAES,MAAMC,UAAU,EAAEP,QAAQ,EAAEQ,UAAU,EAAEjD,aAAa,EAAE,GAAG,MAAMN,OAAOY,IAAI,CAAC,SAAS,IAAM5B,aAAaqD,WAAWZ,UAAU9B,KAAKqB,aAAaoC,OAAOI,IAAI,EAAE;YAAEnC;QAAK;IAC3KX,OAAOJ,aAAa,IAAIA;IACxB8C,OAAOH,MAAM;IACb,KAAK,MAAMQ,OAAOF,WAAYvC,YAAY0C,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAI3C;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjGI,QAAQwC,gBAAgB,CAACD,WAAWf,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiB,kBAAkBF,WAAWxB,MAAM,CAAC,CAAC2B,IAAMtE,yBAAyB4C,GAAG,CAAC0B,MAAM,CAAC/E,iBAAiBqD,GAAG,CAAC0B;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB5D,GAAG,CAACb,YAAY4E,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB5D,GAAG,CAAC,IAAM,KAAK+D,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClL1B,MAAM,CAAC,CAAC2B,IAAMA,MAAM,QACpB7D,GAAG,CAAC,CAAC6D,IAAM,GAAG1E,WAAW0E,GAAG,YAAY,EAAE1E,WAAW0E,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ5B,UAAUF,MAAM,CAAC,CAACP,IAAM,CAACE,SAASM,GAAG,CAACR,EAAEC,OAAO,GAAG5B,GAAG,CAAC,CAAC2B,IAAMA,EAAEC,OAAO;IACpF,MAAMqC,QAAwB;QAAExC;QAAOyC,UAAUb,WAAWrD,GAAG,CAAC,CAACmE,IAAMA,EAAEvC,OAAO;QAAGoC;QAAO/B;IAAS;IACnG,MAAMmC,WAAW,IAAI5E,IAAIwE;IACzB,MAAMK,mBAAmBhB,WAAWrD,GAAG,CAAC,CAACmE,IAAMA,EAAEvC,OAAO,EAAEM,MAAM,CAAC,CAACoC,IAAM,CAACF,SAASjC,GAAG,CAACmC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAItC;WAAaoC;KAAiB;IAElD,OAAO;QACLzB,SAAS;QACTC,QAAQQ,WAAWV,MAAM;QACzBG;QACA,MAAMC,OAAMvC,IAAI;YACd,2FAA2F;YAC3F,yFAAyF;YACzF,MAAMgE,UAAU,MAAMtF,WAAWsB;YACjC,MAAMiE,iBAAiBnB,WAAWpB,MAAM,CAAC,CAACsB,MAAQ,CAACgB,QAAQrC,GAAG,CAACqB;YAC/D,IAAIiB,eAAe9B,MAAM,GAAG,GAAG,MAAM5C,OAAOY,IAAI,CAAC,SAAS,IAAMQ,QAAQuD,UAAU,CAAClE,MAAMiE;YAEzF,IAAIpB,WAAWV,MAAM,GAAG,GAAG;gBACzB,MAAMgC,QAAQ,CAACC,MACbhB,gBAAgB5D,GAAG,CAAC,CAACwD;4BAMZoB;wBALP,IAAIpB,QAAQ,QAAQ,OAAOoB,IAAIhD,OAAO;wBACtC,IAAI4B,QAAQ,UAAU,OAAOoB,IAAIpC,OAAO;wBACxC,IAAIgB,QAAQ,UAAU,OAAOoB,IAAIC,OAAO;wBACxC,IAAIrB,QAAQ,SAAS,OAAOoB,IAAIlC,IAAI;wBACpC,IAAIc,QAAQ,gBAAgB,OAAOoB,IAAIE,UAAU;wBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAACvB,IAAI,cAAboB,2BAAAA,gBAAiB;oBAC1B;gBACF,MAAM7E,OAAOY,IAAI,CAAC,aAAa;oBAC7B,MAAMqE,UAAU3B,WAAWnB,MAAM,CAAC,CAACiC,IAAMC,SAASjC,GAAG,CAACgC,EAAEvC,OAAO;oBAC/D,MAAMqD,aAAa5B,WAAWnB,MAAM,CAAC,CAACiC,IAAM,CAACC,SAASjC,GAAG,CAACgC,EAAEvC,OAAO;oBACnE,MAAM3C,WAAWuB,MAAM,eAAeoD,iBAAiBE,WAAWkB,QAAQhF,GAAG,CAAC2E;oBAC9E,IAAIM,WAAWtC,MAAM,GAAG,GAAG;wBACzB,MAAMuC,OAAOD,WAAWjF,GAAG,CAAC2E;wBAC5B,IAAIxD,QAAQgE,iBAAiB,EAAE,MAAMhE,QAAQgE,iBAAiB,CAAC3E,MAAMoD,iBAAiBsB;6BACjF,MAAM1E,KAAK4E,QAAQ,CAACtB,WAAWoB;oBACtC;gBACF;YACF;YAEA,MAAMnF,OAAOY,IAAI,CAAC,cAAc,IAAMQ,QAAQkE,gBAAgB,CAAC7E,MAAM+D,SAASlB,YAAYY,OAAOvE;YACjG,IAAI6E,QAAQ5B,MAAM,GAAG,GACnB,MAAMnC,KAAK4E,QAAQ,CACjB,gDACAb,QAAQvE,GAAG,CAAC,CAACsE,IAAM;oBAACA;iBAAE;YAE1B,MAAMrF,WACJuB,MACA,mBACA;gBAAC;gBAAQ;aAAO,EAChB,mHACA6C,WAAWrD,GAAG,CAAC,CAAC4E,MAAQ;oBAACA,IAAIhD,OAAO;oBAAEgD,IAAIU,MAAM;iBAAC;YAGnD,MAAMvF,OAAOY,IAAI,CAAC,WAAW;gBAC3B,IAAI4D,QAAQ5B,MAAM,GAAG,GACnB,MAAMnC,KAAK4E,QAAQ,CACjB,6CACAb,QAAQvE,GAAG,CAAC,CAACsE,IAAM;wBAACA;qBAAE;gBAE1B,MAAMiB,aAA0B,EAAE;gBAClC,KAAK,MAAMX,OAAOvB,WAAY,KAAK,MAAMmC,cAAcZ,IAAIa,OAAO,CAAEF,WAAWG,IAAI,CAAC;oBAACd,IAAIhD,OAAO;oBAAE4D;iBAAW;gBAC7G,MAAMvG,WAAWuB,MAAM,gBAAgB;oBAAC;oBAAQ;iBAAS,EAAE,kGAAkG+E;YAC/J;YAEA,IAAItD,SAASU,MAAM,GAAG,GACpB,MAAM5C,OAAOY,IAAI,CAAC,YAAY,IAC5BH,KAAK4E,QAAQ,CACX,4CACAnD,SAASjC,GAAG,CAAC,CAACsE,IAAM;wBAACA;qBAAE;YAI7B,IAAIC,QAAQ5B,MAAM,GAAG,GAAG,KAAK,MAAM1C,WAAWuB,SAAU,MAAMzB,OAAOY,IAAI,CAACvB,aAAaa,QAAQC,IAAI,EAAE,WAAW;oBAAMD;wBAAAA,kBAAAA,QAAQ0F,MAAM,cAAd1F,sCAAAA,qBAAAA,SAAiBO,MAAM+D,SAASN;;YACtJ,KAAK,MAAMhE,WAAWuB,SAAU;gBAC9B,MAAMoE,iBAAiCvC,WAAWrD,GAAG,CAAC,CAAC4E,MAAS,CAAA;wBAAE5C,MAAM4C,IAAIhD,OAAO;wBAAEiE,WAAWjB,IAAIiB,SAAS,CAAC5F,QAAQC,IAAI,CAAC;oBAAC,CAAA;gBAC5H,MAAMH,OAAOY,IAAI,CAACvB,aAAaa,QAAQC,IAAI,EAAE,UAAU;wBAAMD;4BAAAA,iBAAAA,QAAQ6F,KAAK,cAAb7F,qCAAAA,oBAAAA,SAAgBO,MAAMoF,gBAAgB3B;;YACrG;YACA,KAAK,MAAMhE,WAAWuB,SAAU,MAAMzB,OAAOY,IAAI,CAACvB,aAAaa,QAAQC,IAAI,EAAE,UAAU;oBAAMD;wBAAAA,0BAAAA,QAAQ8F,cAAc,cAAtB9F,8CAAAA,6BAAAA,SAAyBO,MAAMyD;;QAC9H;QACAjB,QAAQ,CAAC1C,OAASP,OAAOkD,IAAI,CAAC9C,WAAWG,MAAMG,OAAOJ,aAAa;QACnE,MAAM6C,QAAO1C,IAAI,EAAEF,IAAI;YACrBtB,iBAAiBuC,WAAWjB;YAC5B,IAAIa,QAAQ6E,cAAc,EAAE,MAAMjG,OAAOY,IAAI,CAAC,QAAQ;oBAAMQ;wBAAAA,0BAAAA,QAAQ6E,cAAc,cAAtB7E,8CAAAA,6BAAAA,SAAyBX,MAAMF;;QAC7F;IACF;AACF;AAEA,gGAAgG;AAChG,kGAAkG;AAClG,OAAO,eAAe2F,UAAUzF,IAAgB,EAAEd,GAAW,EAAEwB,KAAqB,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAClK,MAAMZ,SAAShB,sBAAsBC;IACrC,MAAMuB,WAAW,MAAM3B,gBAAgBkB,MAAM,IAAMD,sBAAsBC,MAAMC;IAC/E,MAAMyF,OAAO,MAAMlF,iBAAiBC,UAAUvB,KAAKwB,OAAOC,SAASV,QAAQW,MAAMC;IACjF,IAAI,CAAC6E,KAAKtD,OAAO,EAAE,OAAO;QAAEC,QAAQqD,KAAKrD,MAAM;QAAEC,UAAUoD,KAAKpD,QAAQ;QAAE/C,QAAQmG,KAAKlD,MAAM,CAAC;IAAG;IACjG,MAAMmD,UAAUC,KAAKC,GAAG;IACxB,IAAI/F;IACJ,IAAI;QACF,MAAMhB,gBAAgBkB,MAAM,IAAM0F,KAAKnD,KAAK,CAACvC,OAAOW,QAAQmF,SAAS;IACvE,SAAU;QACRhG,OAAO8F,KAAKC,GAAG,KAAKF;QACpB1F,OAAOH,IAAI,IAAIA;IACjB;IACA,MAAM4F,KAAKhD,MAAM,CAAC1C,MAAMF;IACxB,OAAO;QAAEuC,QAAQqD,KAAKrD,MAAM;QAAEC,UAAUoD,KAAKpD,QAAQ;QAAE/C,QAAQmG,KAAKlD,MAAM,CAACvC,OAAOH,IAAI;IAAE;AAC1F"}

package/dist/esm/store/signature.d.ts

@@ -1,3 +1,4 @@
 export declare function changedSignatureKeys(before: string, after: string): Set<string>;
 export declare function embedIdentityAdopted(before: string, after: string): boolean;
+export declare function embeddingSignatureCompatible(expected: string | null, actual: string): boolean;
 export declare function signatureDiff(before: string, after: string): string;

package/dist/esm/store/signature.js

@@ -27,6 +27,14 @@ export function embedIdentityAdopted(before, after) {
     const at = a.indexOf('@');
     return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
 }
+// Embedding work captured against an unresolved static-model signature may finish under the
+// same model after its weight identity is known. Every published read still requires exactness.
+export function embeddingSignatureCompatible(expected, actual) {
+    if (expected === actual) return true;
+    if (expected === null) return false;
+    const changed = changedSignatureKeys(expected, actual);
+    return changed.size === 1 && changed.has('embed') && embedIdentityAdopted(expected, actual);
+}
 // Names what moved, for the rebuild notice.
 export function signatureDiff(before, after) {
     const changed = changedSignatureKeys(before, after);

package/dist/esm/store/signature.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/signature.ts"],"sourcesContent":["// Feature-signature comparison for open()'s rebuild-vs-adopt decision (config.featureSignature's\n// format: one `feature:<name>:on|off` segment per feature, embed provider, then one segment per\n// preset). Pure string comparison, no SQL, shared by every store's open dialect.\n\n// Segment keys that moved between two feature signatures. `preset:` and `feature:` segments key\n// on their first two colon-separated parts (e.g. `preset:default`, `feature:tags`) so a change to\n// one preset or one feature is independently detectable; every other segment keys on its prefix alone.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') || part.startsWith('feature:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : key.startsWith('feature:') ? `feature \"${key.slice(8)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n"],"names":["changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedIdentityAdopted","embedPart","find","p","undefined","at","indexOf","signatureDiff","label","size"],"mappings":"AAAA,iGAAiG;AACjG,gGAAgG;AAChG,iFAAiF;AAEjF,gGAAgG;AAChG,kGAAkG;AAClG,uGAAuG;AACvG,OAAO,SAASA,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,cAAcD,KAAKC,UAAU,CAAC,cAAcD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACvJ,MAAMG,QAAQ,CAACC,MAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,CAACR,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMS,IAAIJ,MAAMR;IAChB,MAAMa,IAAIL,MAAMP;IAChB,MAAMa,UAAU,IAAIC;IACpB,KAAK,MAAM,CAACC,KAAKC,IAAI,IAAIJ,EAAG,IAAID,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;IAChE,KAAK,MAAMA,OAAOJ,EAAEQ,IAAI,GAAI,IAAI,CAACP,EAAEQ,GAAG,CAACL,MAAMF,QAAQK,GAAG,CAACH;IACzD,OAAOF;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASQ,qBAAqBtB,MAAc,EAAEC,KAAa;IAChE,MAAMsB,YAAY,CAACd,MAAgBA,IAAIJ,KAAK,CAAC,KAAKmB,IAAI,CAAC,CAACC,IAAMA,EAAErB,UAAU,CAAC;IAC3E,MAAMS,IAAIU,UAAUvB;IACpB,MAAMY,IAAIW,UAAUtB;IACpB,IAAIY,MAAMa,aAAad,MAAMc,WAAW,OAAO;IAC/C,MAAMC,KAAKf,EAAEgB,OAAO,CAAC;IACrB,OAAOf,EAAEe,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKf,EAAEN,KAAK,CAAC,GAAGqB,QAAQd;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASgB,cAAc7B,MAAc,EAAEC,KAAa;IACzD,MAAMa,UAAUf,qBAAqBC,QAAQC;IAC7C,MAAM6B,QAAQ,CAACd,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAIZ,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAGU,IAAIZ,UAAU,CAAC,cAAc,CAAC,SAAS,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzL,OAAOQ,QAAQiB,IAAI,KAAK,IAAI,aAAa;WAAIjB;KAAQ,CAACH,GAAG,CAACmB,OAAOvB,IAAI,CAAC;AACxE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/signature.ts"],"sourcesContent":["// Feature-signature comparison for open()'s rebuild-vs-adopt decision (config.featureSignature's\n// format: one `feature:<name>:on|off` segment per feature, embed provider, then one segment per\n// preset). Pure string comparison, no SQL, shared by every store's open dialect.\n\n// Segment keys that moved between two feature signatures. `preset:` and `feature:` segments key\n// on their first two colon-separated parts (e.g. `preset:default`, `feature:tags`) so a change to\n// one preset or one feature is independently detectable; every other segment keys on its prefix alone.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') || part.startsWith('feature:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Embedding work captured against an unresolved static-model signature may finish under the\n// same model after its weight identity is known. Every published read still requires exactness.\nexport function embeddingSignatureCompatible(expected: string | null, actual: string): boolean {\n  if (expected === actual) return true;\n  if (expected === null) return false;\n  const changed = changedSignatureKeys(expected, actual);\n  return changed.size === 1 && changed.has('embed') && embedIdentityAdopted(expected, actual);\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : key.startsWith('feature:') ? `feature \"${key.slice(8)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n"],"names":["changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedIdentityAdopted","embedPart","find","p","undefined","at","indexOf","embeddingSignatureCompatible","expected","actual","size","signatureDiff","label"],"mappings":"AAAA,iGAAiG;AACjG,gGAAgG;AAChG,iFAAiF;AAEjF,gGAAgG;AAChG,kGAAkG;AAClG,uGAAuG;AACvG,OAAO,SAASA,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,cAAcD,KAAKC,UAAU,CAAC,cAAcD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACvJ,MAAMG,QAAQ,CAACC,MAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,CAACR,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMS,IAAIJ,MAAMR;IAChB,MAAMa,IAAIL,MAAMP;IAChB,MAAMa,UAAU,IAAIC;IACpB,KAAK,MAAM,CAACC,KAAKC,IAAI,IAAIJ,EAAG,IAAID,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;IAChE,KAAK,MAAMA,OAAOJ,EAAEQ,IAAI,GAAI,IAAI,CAACP,EAAEQ,GAAG,CAACL,MAAMF,QAAQK,GAAG,CAACH;IACzD,OAAOF;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASQ,qBAAqBtB,MAAc,EAAEC,KAAa;IAChE,MAAMsB,YAAY,CAACd,MAAgBA,IAAIJ,KAAK,CAAC,KAAKmB,IAAI,CAAC,CAACC,IAAMA,EAAErB,UAAU,CAAC;IAC3E,MAAMS,IAAIU,UAAUvB;IACpB,MAAMY,IAAIW,UAAUtB;IACpB,IAAIY,MAAMa,aAAad,MAAMc,WAAW,OAAO;IAC/C,MAAMC,KAAKf,EAAEgB,OAAO,CAAC;IACrB,OAAOf,EAAEe,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKf,EAAEN,KAAK,CAAC,GAAGqB,QAAQd;AAClE;AAEA,4FAA4F;AAC5F,gGAAgG;AAChG,OAAO,SAASgB,6BAA6BC,QAAuB,EAAEC,MAAc;IAClF,IAAID,aAAaC,QAAQ,OAAO;IAChC,IAAID,aAAa,MAAM,OAAO;IAC9B,MAAMhB,UAAUf,qBAAqB+B,UAAUC;IAC/C,OAAOjB,QAAQkB,IAAI,KAAK,KAAKlB,QAAQO,GAAG,CAAC,YAAYC,qBAAqBQ,UAAUC;AACtF;AAEA,4CAA4C;AAC5C,OAAO,SAASE,cAAcjC,MAAc,EAAEC,KAAa;IACzD,MAAMa,UAAUf,qBAAqBC,QAAQC;IAC7C,MAAMiC,QAAQ,CAAClB,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAIZ,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAGU,IAAIZ,UAAU,CAAC,cAAc,CAAC,SAAS,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzL,OAAOQ,QAAQkB,IAAI,KAAK,IAAI,aAAa;WAAIlB;KAAQ,CAACH,GAAG,CAACuB,OAAO3B,IAAI,CAAC;AACxE"}

package/dist/esm/store/sqlite/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { InternalOpenOptions, OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "21";
+export declare const SCHEMA_VERSION = "22";
 export type { OpenResult };
 interface SqliteHandle {
     db: DatabaseSync;

package/dist/esm/store/sqlite/open.js

@@ -1,8 +1,7 @@
 // The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with
 // both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.
 import { DatabaseSync } from 'node:sqlite';
-import { featureSignature } from '../../config/index.js';
-import { activeFeatures, FEATURES } from '../../features/index.js';
+import { activeFeatures } from '../../features/index.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { BEGIN_WRITE, withTransaction } from '../transaction.js';
@@ -13,7 +12,7 @@ import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '21';
+export const SCHEMA_VERSION = '22';
 // unicode61 splits on spaces, so a language written without them indexes a whole run as one
 // token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.
 const TOKENIZE = 'porter unicode61';
@@ -23,8 +22,8 @@ async function createContentTable(conn) {
     await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);
 }
 // Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
-// can't dump file text into context. Features add their own tables after the core ones.
-async function ensureSchemaTables(conn, cfg) {
+// can't dump file text into context. Feature schema is published later with its generation.
+async function ensureCoreSchemaTables(conn) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
     await createContentTable(conn);
     await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources ("path" TEXT PRIMARY KEY, text TEXT NOT NULL)`);
@@ -32,9 +31,7 @@ async function ensureSchemaTables(conn, cfg) {
     // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
-    for (const feature of activeFeatures(cfg))await feature.schema(conn);
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
-    if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
 // Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY
 // with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.
@@ -79,10 +76,12 @@ async function connect(dbPath, _cfg, options) {
 async function close(handle) {
     handle.db.close();
 }
-async function ensureSchema(_handle, conn, cfg) {
-    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
-    // here both see it missing and the second ALTER fails with a duplicate column.
-    await withTransaction(conn, ()=>ensureSchemaTables(conn, cfg), BEGIN_WRITE);
+async function ensureCoreSchema(_handle, conn) {
+    // One writer at a time: two cold opens can bootstrap the same database concurrently.
+    await withTransaction(conn, ()=>ensureCoreSchemaTables(conn), BEGIN_WRITE);
+}
+async function ensureFeatureSchema(_handle, conn, cfg) {
+    for (const feature of activeFeatures(cfg))await feature.schema(conn);
 }
 async function setDerivedBusyTimeout(handle, _conn, ms) {
     handle.db.exec(`PRAGMA busy_timeout = ${ms}`);
@@ -94,7 +93,8 @@ export const sqliteOpenDialect = {
     reconcileDialect: sqliteDialect,
     connect,
     close,
-    ensureSchema,
+    ensureCoreSchema,
+    ensureFeatureSchema,
     setDerivedBusyTimeout,
     createStore: (handle, conn)=>createStore(handle.db, conn)
 };

package/dist/esm/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '21';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath, options.existingOnly ? { readOnly: true } : {});\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    if (!options.observational) setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect, options);\n}\n"],"names":["DatabaseSync","featureSignature","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","TOKENIZE","createContentTable","conn","exec","ensureSchemaTables","cfg","feature","schema","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","options","existingOnly","readOnly","observational","handle","close","ensureSchema","_handle","setDerivedBusyTimeout","_conn","ms","sqliteOpenDialect","filename","schemaVersion","reconcileDialect","openSqlite"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAInC,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMC,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeC,mBAAmBC,IAAgB;IAChD,MAAMA,KAAKC,IAAI,CAAC,CAAC,0IAA0I,EAAEH,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeI,mBAAmBF,IAAgB,EAAEG,GAAW;IAC7D,MAAMH,KAAKC,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,mBAAmBC;IACzB,MAAMA,KAAKC,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,KAAK,MAAMG,WAAWnB,eAAekB,KAAM,MAAMC,QAAQC,MAAM,CAACL;IAChE,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYhB,iBAAiBmB,KAAKjB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASoB,cAAcC,EAAgB;IACrC,MAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGN,IAAI,CAAC;YACR;QACF,EAAE,OAAOU,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB,EAAEC,OAA0D;IACrH,MAAMd,KAAK,IAAIxB,aAAaoC,QAAQE,QAAQC,YAAY,GAAG;QAAEC,UAAU;IAAK,IAAI,CAAC;IACjF,IAAI;QACF,6FAA6F;QAC7F,yFAAyF;QACzFhB,GAAGN,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxE,IAAI,CAACoB,QAAQG,aAAa,EAAElB,cAAcC;QAC1Cb,kBAAkBa;QAClB,MAAMP,OAAOR,iBAAiBe;QAC9B,OAAO;YAAEkB,QAAQ;gBAAElB;YAAG;YAAGP;QAAK;IAChC,EAAE,OAAOW,KAAK;QACZ,4FAA4F;QAC5F,0CAA0C;QAC1CJ,GAAGmB,KAAK;QACR,MAAMf;IACR;AACF;AAEA,eAAee,MAAMD,MAAoB;IACvCA,OAAOlB,EAAE,CAACmB,KAAK;AACjB;AAEA,eAAeC,aAAaC,OAAqB,EAAE5B,IAAgB,EAAEG,GAAW;IAC9E,+FAA+F;IAC/F,+EAA+E;IAC/E,MAAMZ,gBAAgBS,MAAM,IAAME,mBAAmBF,MAAMG,MAAMb;AACnE;AAEA,eAAeuC,sBAAsBJ,MAAoB,EAAEK,KAAiB,EAAEC,EAAU;IACtFN,OAAOlB,EAAE,CAACN,IAAI,CAAC,CAAC,sBAAsB,EAAE8B,IAAI;AAC9C;AAEA,+FAA+F;AAC/F,OAAO,MAAMC,oBAA+C;IAC1DC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkB1C;IAClByB;IACAQ;IACAC;IACAE;IACAlC,aAAa,CAAC8B,QAAQzB,OAASL,YAAY8B,OAAOlB,EAAE,EAAEP;AACxD,EAAE;AAEF,OAAO,eAAeoC,WAAWjC,GAAmB,EAAEkB,OAA6B;IACjF,OAAOlC,gBAAgBgB,KAAK6B,mBAAmBX;AACjD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '22';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Feature schema is published later with its generation.\nasync function ensureCoreSchemaTables(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath, options.existingOnly ? { readOnly: true } : {});\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    if (!options.observational) setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureCoreSchema(_handle: SqliteHandle, conn: Connection): Promise<void> {\n  // One writer at a time: two cold opens can bootstrap the same database concurrently.\n  await withTransaction(conn, () => ensureCoreSchemaTables(conn), BEGIN_WRITE);\n}\n\nasync function ensureFeatureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureCoreSchema,\n  ensureFeatureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect, options);\n}\n"],"names":["DatabaseSync","activeFeatures","openWithDialect","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","TOKENIZE","createContentTable","conn","exec","ensureCoreSchemaTables","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","options","existingOnly","readOnly","observational","handle","close","ensureCoreSchema","_handle","ensureFeatureSchema","cfg","feature","schema","setDerivedBusyTimeout","_conn","ms","sqliteOpenDialect","filename","schemaVersion","reconcileDialect","openSqlite"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,YAAY,QAAQ,cAAc;AAE3C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAInC,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMC,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeC,mBAAmBC,IAAgB;IAChD,MAAMA,KAAKC,IAAI,CAAC,CAAC,0IAA0I,EAAEH,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,eAAeI,uBAAuBF,IAAgB;IACpD,MAAMA,KAAKC,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,mBAAmBC;IACzB,MAAMA,KAAKC,IAAI,CAAC,CAAC,wFAAwF,CAAC;IAC1G,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,IAAI,AAAC,MAAMb,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;AAC9F;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASM,cAAcC,EAAgB;IACrC,MAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGH,IAAI,CAAC;YACR;QACF,EAAE,OAAOO,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB,EAAEC,OAA0D;IACrH,MAAMd,KAAK,IAAInB,aAAa+B,QAAQE,QAAQC,YAAY,GAAG;QAAEC,UAAU;IAAK,IAAI,CAAC;IACjF,IAAI;QACF,6FAA6F;QAC7F,yFAAyF;QACzFhB,GAAGH,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxE,IAAI,CAACiB,QAAQG,aAAa,EAAElB,cAAcC;QAC1CV,kBAAkBU;QAClB,MAAMJ,OAAOR,iBAAiBY;QAC9B,OAAO;YAAEkB,QAAQ;gBAAElB;YAAG;YAAGJ;QAAK;IAChC,EAAE,OAAOQ,KAAK;QACZ,4FAA4F;QAC5F,0CAA0C;QAC1CJ,GAAGmB,KAAK;QACR,MAAMf;IACR;AACF;AAEA,eAAee,MAAMD,MAAoB;IACvCA,OAAOlB,EAAE,CAACmB,KAAK;AACjB;AAEA,eAAeC,iBAAiBC,OAAqB,EAAEzB,IAAgB;IACrE,qFAAqF;IACrF,MAAMT,gBAAgBS,MAAM,IAAME,uBAAuBF,OAAOV;AAClE;AAEA,eAAeoC,oBAAoBD,OAAqB,EAAEzB,IAAgB,EAAE2B,GAAW;IACrF,KAAK,MAAMC,WAAW1C,eAAeyC,KAAM,MAAMC,QAAQC,MAAM,CAAC7B;AAClE;AAEA,eAAe8B,sBAAsBR,MAAoB,EAAES,KAAiB,EAAEC,EAAU;IACtFV,OAAOlB,EAAE,CAACH,IAAI,CAAC,CAAC,sBAAsB,EAAE+B,IAAI;AAC9C;AAEA,+FAA+F;AAC/F,OAAO,MAAMC,oBAA+C;IAC1DC,UAAUtC;IACVuC,eAAetC;IACfuC,kBAAkB3C;IAClBsB;IACAQ;IACAC;IACAE;IACAI;IACAnC,aAAa,CAAC2B,QAAQtB,OAASL,YAAY2B,OAAOlB,EAAE,EAAEJ;AACxD,EAAE;AAEF,OAAO,eAAeqC,WAAWV,GAAmB,EAAET,OAA6B;IACjF,OAAO/B,gBAAgBwC,KAAKM,mBAAmBf;AACjD"}

package/dist/esm/store/sqlite/reconcile.js

@@ -26,8 +26,8 @@ export function contentRow(doc) {
 async function reconcileContent(conn, touched, docs, _delta, _cfg) {
     // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via
     // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.
-    // `touched` already covers a path a concurrent reconcile turned out to have created first --
-    // see reconcile.ts's post-lock revalidation, which folds such paths in before calling here.
+    // builder.ts's generation fence rejects a plan whose added/touched classification went stale
+    // behind another publication, so this receives one classification from the locked generation.
     if (touched.length > 0) await conn.runBatch('DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE "path" = ?)', touched.map((p)=>[
             p
         ]));