sensemaking 0.18.1 → 0.18.3

package/dist/esm/store/turso/reconcile.js

@@ -0,0 +1,158 @@
+import { SenseError } from '../../errors.js';
+import { activeFeatures } from '../../features/index.js';
+import { progress } from '../../output/progress.js';
+import { listFiles, RESERVED_COLUMNS } from '../../scan/index.js';
+import { reparseFiles } from '../../scan/reparse.js';
+import { getColumns, getMeta, quoteIdent, setMeta } from '../shared.js';
+import { withTransaction } from '../transaction.js';
+// Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not
+// duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb
+// dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)
+// is derived from exactly that recorded value -- forking duckdb's file would lose it silently.
+// content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index
+// (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not
+// FTS5-shaped).
+const CORE_FRONTMATTER_COLUMNS = new Set([
+    'path',
+    '_mtime',
+    '_ctime',
+    '_size',
+    '_parse_error'
+]);
+// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no
+// error). The real fence is a SELECT projecting more than this many result columns, which fails
+// to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table
+// would pass every write silently and only break the first time something selects across it.
+// Enforced here, before ALTER, the same shape as sqlite's compile-time fence.
+const MAX_FRONTMATTER_COLUMNS = 2000;
+// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
+const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)`;
+function contentRow(doc) {
+    return [
+        doc.relPath,
+        doc.search.title,
+        doc.search.summary,
+        doc.search.text
+    ];
+}
+export async function reconcile(conn, cfg, baseDir) {
+    const files = listFiles(cfg, baseDir);
+    const currentSet = new Set(files.map((f)=>f.relPath));
+    const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
+    const existingRows = await existingStmt.all();
+    const existing = new Map(existingRows.map((r)=>[
+            r.path,
+            r
+        ]));
+    const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
+    const toReparse = files.filter((f)=>{
+        const row = existing.get(f.relPath);
+        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
+    });
+    if (vanished.length === 0 && toReparse.length === 0) return {
+        parsed: 0,
+        warnings: []
+    };
+    const features = activeFeatures(cfg);
+    const seenColumns = await getColumns(conn);
+    // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
+    // reconciles stay silent (progress() has a threshold).
+    const report = progress('reparsing files', toReparse.length);
+    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    report.finish();
+    for (const col of newColumns)seenColumns.add(col);
+    const allColumns = [
+        ...seenColumns
+    ];
+    if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {
+        throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
+    }
+    // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a
+    // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).
+    const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
+    // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.
+    const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
+    const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
+    const delta = {
+        files,
+        reparsed: parsedDocs.map((d)=>d.relPath),
+        added,
+        vanished
+    };
+    const addedSet = new Set(added);
+    const reparsedExisting = parsedDocs.map((d)=>d.relPath).filter((p)=>!addedSet.has(p));
+    const txStart = Date.now();
+    await withTransaction(conn, async ()=>{
+        var _feature_remove, _feature_afterReconcile;
+        for (const col of newColumns)await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);
+        if (parsedDocs.length > 0) {
+            const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
+                    var _doc_data_col;
+                    if (col === 'path') return doc.relPath;
+                    if (col === '_mtime') return doc.mtimeMs;
+                    if (col === '_ctime') return doc.ctimeMs;
+                    if (col === '_size') return doc.size;
+                    // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
+                    if (col === '_parse_error') return doc.parseError;
+                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
+                }));
+            await conn.runBatch(insertSql, rows);
+        }
+        // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5
+        // content), so vanished and reparsed docs delete in one pass.
+        const contentTouched = [
+            ...vanished,
+            ...reparsedExisting
+        ];
+        if (contentTouched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', contentTouched.map((p)=>[
+                p
+            ]));
+        if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));
+        if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
+                p
+            ]));
+        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
+        // new docs have nothing to clear, which keeps cold builds linear.
+        const presetTouched = [
+            ...vanished,
+            ...reparsedExisting
+        ];
+        if (presetTouched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', presetTouched.map((p)=>[
+                p
+            ]));
+        const presetRows = [];
+        for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
+            doc.relPath,
+            presetName
+        ]);
+        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
+        const removedPaths = [
+            ...vanished,
+            ...reparsedExisting
+        ];
+        if (removedPaths.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, removedPaths, delta));
+        for (const feature of features){
+            var _feature_store;
+            const docsForFeature = parsedDocs.map((doc)=>({
+                    path: doc.relPath,
+                    extracted: doc.extracted[feature.name]
+                }));
+            await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
+        }
+        for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
+    });
+    // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the
+    // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait
+    // against (the same mechanism sqlite's reconcile records, and the one this store's busy
+    // timeout is derived from).
+    const durationMs = Date.now() - txStart;
+    const prevRaw = await getMeta(conn, 'reconcile_max_ms');
+    // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
+    // still gets recorded instead of losing to the "nothing recorded yet" default.
+    const prevMax = prevRaw === null ? -1 : Number(prevRaw);
+    if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));
+    return {
+        parsed: parsedDocs.length,
+        warnings
+    };
+}

package/dist/esm/store/turso/reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not\n// duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb\n// dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)\n// is derived from exactly that recorded value -- forking duckdb's file would lose it silently.\n// content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index\n// (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not\n// FTS5-shaped).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no\n// error). The real fence is a SELECT projecting more than this many result columns, which fails\n// to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table\n// would pass every write silently and only break the first time something selects across it.\n// Enforced here, before ALTER, the same shape as sqlite's compile-time fence.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n    // content), so vanished and reparsed docs delete in one pass.\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n  // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait\n  // against (the same mechanism sqlite's reconcile records, and the one this store's busy\n  // timeout is derived from).\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,8FAA8F;AAC9F,gGAAgG;AAChG,kGAAkG;AAClG,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,gBAAgB;AAChB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,iGAAiG;AACjG,gGAAgG;AAChG,iGAAiG;AACjG,6FAA6F;AAC7F,8EAA8E;AAC9E,MAAMC,0BAA0B;AAEhC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQvB,UAAUqB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWlD,eAAeuB;IAChC,MAAM4B,cAAc,MAAM9C,WAAWiB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAASnD,SAAS,mBAAmBuC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMnD,aAAaoC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAGnC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAE6D,WAAWb,MAAM,CAAC,2DAA2D,EAAEnC,wBAAwB,gPAAgP,CAAC;IAEtX;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAAC3D,iBAAiBoC,GAAG,CAACuB;IAC1G,uFAAuF;IACvF,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACpB,YAAYyD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGvD,WAAWuD,GAAG,YAAY,EAAEvD,WAAWuD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMjE,gBAAgBa,MAAM;YA+CyCqD,iBAK7BA;QAnDtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAErE,WAAWmD,MAAM;QAErG,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM8B,OAAOvB,WAAW3B,GAAG,CAAC,CAACZ,MAC3B8C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ3C;oBANP,IAAI2C,QAAQ,QAAQ,OAAO3C,IAAIC,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO3C,IAAI6B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO3C,IAAI+D,OAAO;oBACxC,IAAIpB,QAAQ,SAAS,OAAO3C,IAAI+B,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO3C,IAAIgE,UAAU;oBACjD,QAAOhE,gBAAAA,IAAIiE,IAAI,CAACtB,IAAI,cAAb3C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMO,KAAK2D,QAAQ,CAAClB,WAAWc;QACjC;QAEA,0FAA0F;QAC1F,8DAA8D;QAC9D,MAAMK,iBAAiB;eAAI7C;eAAaiC;SAAiB;QACzD,IAAIY,eAAenC,MAAM,GAAG,GAC1B,MAAMzB,KAAK2D,QAAQ,CACjB,wCACAC,eAAevD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAACpE,oBAAoByC,WAAW3B,GAAG,CAACb;QAElF,IAAIuB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAK2D,QAAQ,CACjB,4CACA5C,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAK2D,QAAQ,CACjB,6CACAE,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMrE,OAAOuC,WAAY,KAAK,MAAM+B,cAActE,IAAIuE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACxE,IAAIC,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACZ,MAAS,CAAA;oBAAEqB,MAAMrB,IAAIC,OAAO;oBAAE2E,WAAW5E,IAAI4E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,0FAA0F;IAC1F,4FAA4F;IAC5F,wFAAwF;IACxF,4BAA4B;IAC5B,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM1F,QAAQgB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAMzF,QAAQc,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/turso/store.d.ts

@@ -0,0 +1,5 @@
+import type { Database } from '@tursodatabase/database';
+import type { Config } from '../../config/index.js';
+import type { Capability, Connection, Store } from '../types.js';
+export declare const CAPABILITIES: ReadonlySet<Capability>;
+export declare function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store;

package/dist/esm/store/turso/store.js

@@ -0,0 +1,93 @@
+import { SenseError } from '../../errors.js';
+import { getColumns } from '../shared.js';
+import { withTransaction } from '../transaction.js';
+import { hasVectorRow, pendingRows } from '../vectors.js';
+import { fieldStats } from './fieldStats.js';
+import { reconcile } from './reconcile.js';
+// Phase 1: the portable surface only -- frontmatter, links, sections, tags, rank, raw SQL,
+// reconcile, transactions. Neither lexical (Tantivy FTS) nor vectors (native vector_distance_cos
+// scans) is claimed yet -- phase 2 and phase 3 of the plan add them -- so lexical.query() and the
+// vector-math methods throw the named capability error rather than answer differently.
+// pending()/hasVector() are plain IS NULL/IS NOT NULL checks with no engine-specific math, so
+// they're wired through the same store-agnostic helpers duckdb's store.ts reuses: harmless ahead
+// of phase 3 (unreachable in practice, since openStore() already rejects a turso tree whose
+// config uses vectors) and correct with no further change once vectors land.
+export const CAPABILITIES = new Set();
+function missing(name) {
+    throw new SenseError('STORE_CAPABILITY_MISSING', `store "turso" does not implement "${name}" in this build; set "store" to "sqlite" or "duckdb" in this tree's config`);
+}
+// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth
+// (see transaction.ts) is tracked against the same object everywhere.
+export function createStore(db, conn, cfg, baseDir) {
+    return {
+        name: 'turso',
+        capabilities: CAPABILITIES,
+        async exec (sql) {
+            await conn.exec(sql);
+        },
+        async prepare (sql) {
+            return conn.prepare(sql);
+        },
+        async runBatch (sql, paramRows) {
+            await conn.runBatch(sql, paramRows);
+        },
+        async transaction (fn) {
+            return withTransaction(conn, fn);
+        },
+        async reconcile () {
+            return reconcile(conn, cfg, baseDir);
+        },
+        docs: {
+            async columns () {
+                return [
+                    ...await getColumns(conn)
+                ];
+            },
+            fieldStats: (columns, scopeWhere)=>fieldStats(conn, columns, scopeWhere)
+        },
+        lexical: {
+            async query () {
+                return missing('lexical');
+            }
+        },
+        vectors: {
+            pending: ()=>pendingRows(conn),
+            async writeVectors () {
+                return missing('vectors');
+            },
+            async candidates () {
+                return missing('vectors');
+            },
+            async similar () {
+                return missing('vectors');
+            },
+            hasVector: (path)=>hasVectorRow(conn, path)
+        },
+        async engineStatus () {
+            // Read back rather than recomputed: this is what open() actually set (3x the largest
+            // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.
+            // Turso's raw PRAGMA busy_timeout names its result column "busy_timeout" (spike-verified),
+            // not "timeout" like real SQLite's own quirky column name for this one pragma.
+            const row = await (await db.prepare('PRAGMA busy_timeout')).get();
+            return {
+                busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)`
+            };
+        },
+        raw: {
+            async prepare (sql) {
+                const stmt = await db.prepare(sql);
+                stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision
+                return {
+                    columns: ()=>stmt.columns(),
+                    // sense sql streams through the client's own async generator.
+                    iterate: async function*(...params) {
+                        yield* stmt.iterate(...params);
+                    }
+                };
+            }
+        },
+        async close () {
+            await db.close();
+        }
+    };
+}

package/dist/esm/store/turso/store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { reconcile } from './reconcile.ts';\n\n// Phase 1: the portable surface only -- frontmatter, links, sections, tags, rank, raw SQL,\n// reconcile, transactions. Neither lexical (Tantivy FTS) nor vectors (native vector_distance_cos\n// scans) is claimed yet -- phase 2 and phase 3 of the plan add them -- so lexical.query() and the\n// vector-math methods throw the named capability error rather than answer differently.\n// pending()/hasVector() are plain IS NULL/IS NOT NULL checks with no engine-specific math, so\n// they're wired through the same store-agnostic helpers duckdb's store.ts reuses: harmless ahead\n// of phase 3 (unreachable in practice, since openStore() already rejects a turso tree whose\n// config uses vectors) and correct with no further change once vectors land.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set();\n\nfunction missing(name: 'lexical' | 'vectors'): never {\n  throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement \"${name}\" in this build; set \"store\" to \"sqlite\" or \"duckdb\" in this tree's config`);\n}\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      async query() {\n        return missing('lexical');\n      },\n    },\n    vectors: {\n      pending: () => pendingRows(conn),\n      async writeVectors() {\n        return missing('vectors');\n      },\n      async candidates() {\n        return missing('vectors');\n      },\n      async similar() {\n        return missing('vectors');\n      },\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      // Turso's raw PRAGMA busy_timeout names its result column \"busy_timeout\" (spike-verified),\n      // not \"timeout\" like real SQLite's own quirky column name for this one pragma.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["SenseError","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","reconcile","CAPABILITIES","Set","missing","name","createStore","db","conn","cfg","baseDir","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","candidates","similar","hasVector","path","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AAEA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,SAAS,QAAQ,iBAAiB;AAE3C,2FAA2F;AAC3F,iGAAiG;AACjG,kGAAkG;AAClG,uFAAuF;AACvF,8FAA8F;AAC9F,iGAAiG;AACjG,4FAA4F;AAC5F,6EAA6E;AAC7E,OAAO,MAAMC,eAAwC,IAAIC,MAAM;AAE/D,SAASC,QAAQC,IAA2B;IAC1C,MAAM,IAAIV,WAAW,4BAA4B,CAAC,kCAAkC,EAAEU,KAAK,0EAA0E,CAAC;AACxK;AAEA,8FAA8F;AAC9F,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,OAAO;QACLL,MAAM;QACNM,cAAcT;QACd,MAAMU,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOrB,gBAAgBW,MAAMU;QAC/B;QACA,MAAMjB;YACJ,OAAOA,UAAUO,MAAMC,KAAKC;QAC9B;QACAS,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMxB,WAAWY;iBAAO;YACtC;YACAR,YAAY,CAACoB,SAASC,aAAerB,WAAWQ,MAAMY,SAASC;QACjE;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOnB,QAAQ;YACjB;QACF;QACAoB,SAAS;YACPC,SAAS,IAAM1B,YAAYS;YAC3B,MAAMkB;gBACJ,OAAOtB,QAAQ;YACjB;YACA,MAAMuB;gBACJ,OAAOvB,QAAQ;YACjB;YACA,MAAMwB;gBACJ,OAAOxB,QAAQ;YACjB;YACAyB,WAAW,CAACC,OAAShC,aAAaU,MAAMsB;QAC1C;QACA,MAAMC;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,2FAA2F;YAC3F,+EAA+E;YAC/E,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAMzB,GAAGO,OAAO,CAAC,sBAAqB,EAAGmB,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAMrB,SAAQD,GAAW;gBACvB,MAAMuB,OAAO,MAAM7B,GAAGO,OAAO,CAACD;gBAC9BuB,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLjB,SAAS,IAAMgB,KAAKhB,OAAO;oBAC3B,8DAA8D;oBAC9DkB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMjC,GAAGiC,KAAK;QAChB;IACF;AACF"}

package/dist/esm/store/types.d.ts

@@ -1,3 +1,4 @@
+import type { StoreName } from '../config/index.js';
 export type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';
 export interface RunResult {
     changes: number | bigint;
@@ -18,8 +19,14 @@ export interface Connection {
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
 }
+export interface FieldStat {
+    field: string;
+    coverage: number;
+    type: string;
+}
 export interface DocumentStore {
     columns(): Promise<string[]>;
+    fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;
 }
 export interface LexicalHit {
     path: string;
@@ -73,7 +80,7 @@ export interface SqlSession {
     prepare(sql: string): Promise<RawStatement>;
 }
 export interface Store {
-    readonly name: 'sqlite' | 'duckdb';
+    readonly name: StoreName;
     readonly capabilities: ReadonlySet<Capability>;
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
@@ -87,5 +94,6 @@ export interface Store {
     lexical: LexicalIndex;
     vectors: VectorStore;
     raw: SqlSession;
+    engineStatus(): Promise<Record<string, string>>;
     close(): Promise<void>;
 }

package/dist/esm/store/types.js

@@ -1,11 +1,4 @@
 // The backing-store interface: a minimal portable statement surface (exec/prepare, used as-is
 // by feature-owned tables and ad hoc queries) plus dedicated interfaces exactly where engines
 // diverge (lexical index, vector scan, raw sql passthrough).
-// 'lexical'/'vectors' mean the store's LexicalIndex/VectorStore are functionally implemented
-// (rather than present-but-inert); 'phrases'/'snippets'/'watch-concurrency' are the finer
-// behaviors sqlite's FTS5 path carries. A tree whose config needs a capability the chosen
-// store lacks fails at open (or at first use for a lazily-detected need), named.
-// 'phrases' means quoted-phrase (`"..."`) matching only -- not FTS5's wider query grammar
-// (prefix `*`, boolean AND/OR/NOT, NEAR, `^`, column filters), which duckdb's lexical path
-// rejects loudly rather than interpret (see store/duckdb/lexical.ts).
 export { };

package/dist/esm/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare, used as-is\n// by feature-owned tables and ad hoc queries) plus dedicated interfaces exactly where engines\n// diverge (lexical index, vector scan, raw sql passthrough).\n\n// 'lexical'/'vectors' mean the store's LexicalIndex/VectorStore are functionally implemented\n// (rather than present-but-inert); 'phrases'/'snippets'/'watch-concurrency' are the finer\n// behaviors sqlite's FTS5 path carries. A tree whose config needs a capability the chosen\n// store lacks fails at open (or at first use for a lazily-detected need), named.\n// 'phrases' means quoted-phrase (`\"...\"`) matching only -- not FTS5's wider query grammar\n// (prefix `*`, boolean AND/OR/NOT, NEAR, `^`, column filters), which duckdb's lexical path\n// rejects loudly rather than interpret (see store/duckdb/lexical.ts).\nexport type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every engine this store supports has to cross a real\n// async boundary for a query (DuckDB has no synchronous client at all), so run/get/all return\n// Promises. iterate() stays an async iterable for the same reason `sense sql` streams.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's own hot loops (schema,\n// reconcile). Portable so a feature never imports an engine-specific client type. `runBatch`\n// is the one call a write loop goes through instead of preparing once and calling `run()` per\n// row itself: one crossing per loop, the engine's own bulk idiom underneath (see each store's\n// implementation for what that idiom is).\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is still NULL (never embedded, or added since).\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: 'sqlite' | 'duckdb';\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, the graph\n  // ring's temp-table writes, etc.) goes through this instead of looping over `run()` itself.\n  // See Connection.runBatch -- same contract, same per-store implementation.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A command whose\n  // scope holds a network-bound write (search's embedding top-up) must stay outside it.\n  // Nesting joins the enclosing transaction rather than opening a second one.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  close(): Promise<void>;\n}\n"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,8FAA8F;AAC9F,6DAA6D;AAE7D,6FAA6F;AAC7F,0FAA0F;AAC1F,0FAA0F;AAC1F,iFAAiF;AACjF,0FAA0F;AAC1F,2FAA2F;AAC3F,sEAAsE;AA8FtE,WAqBC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare, used as-is\n// by feature-owned tables and ad hoc queries) plus dedicated interfaces exactly where engines\n// diverge (lexical index, vector scan, raw sql passthrough).\n\nimport type { StoreName } from '../config/index.ts';\n\n// 'lexical'/'vectors' mean the store's LexicalIndex/VectorStore are functionally implemented\n// (rather than present-but-inert); 'phrases'/'snippets'/'watch-concurrency' are the finer\n// behaviors sqlite's FTS5 path carries. A tree whose config needs a capability the chosen\n// store lacks fails at open (or at first use for a lazily-detected need), named.\n// 'phrases' means quoted-phrase (`\"...\"`) matching only -- not FTS5's wider query grammar\n// (prefix `*`, boolean AND/OR/NOT, NEAR, `^`, column filters), which duckdb's lexical path\n// rejects loudly rather than interpret (see store/duckdb/lexical.ts).\nexport type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every engine this store supports has to cross a real\n// async boundary for a query (DuckDB has no synchronous client at all), so run/get/all return\n// Promises. iterate() stays an async iterable for the same reason `sense sql` streams.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's own hot loops (schema,\n// reconcile). Portable so a feature never imports an engine-specific client type. `runBatch`\n// is the one call a write loop goes through instead of preparing once and calling `run()` per\n// row itself: one crossing per loop, the engine's own bulk idiom underneath (see each store's\n// implementation for what that idiom is).\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in SQL -- one query\n  // per call, never one row per note. `columns` is one chunk (caller owns the chunk size, a\n  // result-row column limit); `scopeWhere` is a caller-built WHERE fragment, same convention as\n  // LexicalQueryOptions. Each store expresses \"observed type\" through its own engine mechanism\n  // (native-not-emulated) but returns the shared integer/real/text vocabulary.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is still NULL (never embedded, or added since).\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, the graph\n  // ring's temp-table writes, etc.) goes through this instead of looping over `run()` itself.\n  // See Connection.runBatch -- same contract, same per-store implementation.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A command whose\n  // scope holds a network-bound write (search's embedding top-up) must stay outside it.\n  // Nesting joins the enclosing transaction rather than opening a second one.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading), each\n  // store owning what it reports and how it is worded; the command prints entries generically,\n  // one line per fact, without knowing any store's name. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,8FAA8F;AAC9F,6DAA6D;AAoH7D,WAyBC"}

package/dist/esm/store/vectors.d.ts

@@ -0,0 +1,9 @@
+import type { Connection } from './types.js';
+export declare const TARGET_CHUNK_CAP = 16;
+export declare function sampleEvenly<T>(rows: T[], cap?: number): T[];
+export declare function asCosine(score: number): number;
+export declare function pendingRows(conn: Connection): Promise<Array<{
+    path: string;
+    chunk: number;
+}>>;
+export declare function hasVectorRow(conn: Connection, path: string): Promise<boolean>;

package/dist/esm/store/vectors.js

@@ -0,0 +1,26 @@
+// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a
+// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).
+export const TARGET_CHUNK_CAP = 16;
+// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote
+// instead of being cut off by a fixed prefix.
+export function sampleEvenly(rows, cap = TARGET_CHUNK_CAP) {
+    const step = Math.max(1, Math.ceil(rows.length / cap));
+    return rows.filter((_, i)=>i % step === 0);
+}
+// Dequantised int8 dot products (sqlite) land a little either side of a true cosine, so an
+// identical pair can print 1.001; array_cosine_similarity (duckdb) has no such error but is
+// rounded the same way, so both stores print the same bounded number.
+export function asCosine(score) {
+    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
+}
+export async function pendingRows(conn) {
+    const stmt = await conn.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk');
+    return await stmt.all();
+}
+// Whether a note has any chunk with a vector. Distinguishes "nothing is near this note" from
+// "this note has no text", which look the same in an empty result.
+export async function hasVectorRow(conn, path) {
+    const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE "path" = ? AND vector IS NOT NULL LIMIT 1');
+    const row = await stmt.get(path);
+    return row !== undefined;
+}

package/dist/esm/store/vectors.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Dequantised int8 dot products (sqlite) land a little either side of a true cosine, so an\n// identical pair can print 1.001; array_cosine_similarity (duckdb) has no such error but is\n// rounded the same way, so both stores print the same bounded number.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","pendingRows","conn","stmt","prepare","all","hasVectorRow","path","row","get","undefined"],"mappings":"AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMA,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,sEAAsE;AACtE,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,OAAO,eAAeG,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEK,IAAY;IAC/D,MAAMJ,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMI,MAAO,MAAML,KAAKM,GAAG,CAACF;IAC5B,OAAOC,QAAQE;AACjB"}

package/dist/esm/watch.js

@@ -44,27 +44,38 @@ export async function runWatch(cfg, opts = {}) {
     };
     await touchHeartbeat();
     let debounceTimer = null;
+    // A reconcile that has already started owns the store, and on a bulk reparse a live worker
+    // pool as well. Shutdown waits on this rather than closing the connection underneath it.
+    let inFlight = null;
     const scheduleReconcile = ()=>{
         if (debounceTimer) clearTimeout(debounceTimer);
-        debounceTimer = setTimeout(async ()=>{
+        debounceTimer = setTimeout(()=>{
             debounceTimer = null;
-            try {
-                const { parsed, warnings } = await store.reconcile();
-                onEvent({
-                    type: 'reconciled',
-                    parsed,
-                    total: await docCount(store),
-                    warnings
-                });
-            } catch (err) {
-                onEvent({
-                    type: 'reconcile-error',
-                    message: err.message
-                });
-            }
+            inFlight = (async ()=>{
+                try {
+                    const { parsed, warnings } = await store.reconcile();
+                    onEvent({
+                        type: 'reconciled',
+                        parsed,
+                        total: await docCount(store),
+                        warnings
+                    });
+                } catch (err) {
+                    onEvent({
+                        type: 'reconcile-error',
+                        message: err.message
+                    });
+                } finally{
+                    inFlight = null;
+                }
+            })();
         }, debounceMs);
     };
-    // Ignore our own state dir, or the heartbeat write would retrigger itself forever.
+    // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event
+    // whose filename the platform could not resolve (null, which fs.watch does deliver under
+    // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs
+    // less than missing a real edit. That is why the guard cannot promise zero reconciles, only
+    // that an identified state-dir write is never one of them.
     const watcher = fsWatch(baseDir, {
         recursive: true
     }, (_event, filename)=>{
@@ -86,6 +97,7 @@ export async function runWatch(cfg, opts = {}) {
             clearInterval(heartbeatTimer);
             if (debounceTimer) clearTimeout(debounceTimer);
             watcher.close();
+            await inFlight;
             await setMeta(store, 'watch_heartbeat', null);
             await setMeta(store, 'watch_pid', null);
             await store.close();

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(async () => {\n      debounceTimer = null;\n      try {\n        const { parsed, warnings } = await store.reconcile();\n        onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAElG,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUO;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMpB,QAAQa,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMhC,SAASc;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,MAAMC,oBAAoB;QACxB,IAAID,eAAeE,aAAaF;QAChCA,gBAAgBG,WAAW;YACzBH,gBAAgB;YAChB,IAAI;gBACF,MAAM,EAAErB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM6B,SAAS;gBAClDhC,QAAQ;oBAAEmB,MAAM;oBAAcZ;oBAAQc,OAAO,MAAMhC,SAASc;oBAAQE;gBAAS;YAC/E,EAAE,OAAO4B,KAAK;gBACZjC,QAAQ;oBAAEmB,MAAM;oBAAmBe,SAAS,AAACD,IAAcC,OAAO;gBAAC;YACrE;QACF,GAAGjC;IACL;IAEA,mFAAmF;IACnF,MAAMkC,UAAUlD,QAAQwB,SAAS;QAAE2B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACrD,YAAY;QACpE2C;IACF;IACA,MAAMW,iBAAiBC,YACrBrD,YAAYmC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIwC,QAAc,CAACC;YAmBpB5C,cACCA;QAnBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM6C,WAAW;gBAKf7C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQmB,GAAG,CAAC,UAAUD;YACtBlB,QAAQmB,GAAG,CAAC,WAAWD;aACvB7C,eAAAA,KAAK+C,MAAM,cAAX/C,mCAAAA,aAAagD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIZ,eAAeE,aAAaF;YAChCO,QAAQnB,KAAK;YACb,MAAMvB,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB2B;QACF;QACAjB,QAAQuB,IAAI,CAAC,UAAUL;QACvBlB,QAAQuB,IAAI,CAAC,WAAWL;QACxB,KAAI7C,eAAAA,KAAK+C,MAAM,cAAX/C,mCAAAA,aAAamD,OAAO,EAAEN;cACrB7C,gBAAAA,KAAK+C,MAAM,cAAX/C,oCAAAA,cAAaoD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event\n  // whose filename the platform could not resolve (null, which fs.watch does deliver under\n  // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs\n  // less than missing a real edit. That is why the guard cannot promise zero reconciles, only\n  // that an identified state-dir write is never one of them.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","inFlight","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAElG,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUO;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMpB,QAAQa,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMhC,SAASc;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAIC,WAAiC;IACrC,MAAMC,oBAAoB;QACxB,IAAIF,eAAeG,aAAaH;QAChCA,gBAAgBI,WAAW;YACzBJ,gBAAgB;YAChBC,WAAW,AAAC,CAAA;gBACV,IAAI;oBACF,MAAM,EAAEtB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM8B,SAAS;oBAClDjC,QAAQ;wBAAEmB,MAAM;wBAAcZ;wBAAQc,OAAO,MAAMhC,SAASc;wBAAQE;oBAAS;gBAC/E,EAAE,OAAO6B,KAAK;oBACZlC,QAAQ;wBAAEmB,MAAM;wBAAmBgB,SAAS,AAACD,IAAcC,OAAO;oBAAC;gBACrE,SAAU;oBACRN,WAAW;gBACb;YACF,CAAA;QACF,GAAG5B;IACL;IAEA,4FAA4F;IAC5F,yFAAyF;IACzF,4FAA4F;IAC5F,4FAA4F;IAC5F,2DAA2D;IAC3D,MAAMmC,UAAUnD,QAAQwB,SAAS;QAAE4B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACtD,YAAY;QACpE4C;IACF;IACA,MAAMW,iBAAiBC,YACrBtD,YAAYmC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIyC,QAAc,CAACC;YAoBpB7C,cACCA;QApBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM8C,WAAW;gBAKf9C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQoB,GAAG,CAAC,UAAUD;YACtBnB,QAAQoB,GAAG,CAAC,WAAWD;aACvB9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaiD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIb,eAAeG,aAAaH;YAChCQ,QAAQpB,KAAK;YACb,MAAMa;YACN,MAAMpC,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB4B;QACF;QACAlB,QAAQwB,IAAI,CAAC,UAAUL;QACvBnB,QAAQwB,IAAI,CAAC,WAAWL;QACxB,KAAI9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaoD,OAAO,EAAEN;cACrB9C,gBAAAA,KAAKgD,MAAM,cAAXhD,oCAAAA,cAAaqD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}

package/dist/esm/workers/parse.d.ts

@@ -0,0 +1,18 @@
+import type { Config, FeatureName } from '../config/index.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from '../scan/list.js';
+import type { WorkerErrorPayload } from '../scan/worker-error.js';
+export interface ParseWorkerData {
+    cfg: Config;
+    featureNames: FeatureName[];
+}
+export type ParseTask = FileStat;
+export type ParseTaskResult = {
+    ok: true;
+    doc: ParsedDoc;
+    warnings: string[];
+} | {
+    ok: false;
+    error: WorkerErrorPayload;
+};
+export default function parseTask(file: ParseTask): ParseTaskResult;

package/dist/esm/workers/parse.js

@@ -0,0 +1,25 @@
+import Tinypool from 'tinypool';
+import { FEATURES } from '../features/index.js';
+import { parseFile } from '../scan/index.js';
+import { featuresForFile } from '../scan/reparse.js';
+import { serializeError } from '../scan/worker-error.js';
+// Read once per worker, not per task.
+const { cfg, featureNames } = Tinypool.workerData;
+// Filtering the registry (rather than mapping the names) keeps registry order, which is the
+// order `extracted` keys land in on the serial path.
+const selected = FEATURES.filter((feature)=>featureNames.includes(feature.name));
+export default function parseTask(file) {
+    try {
+        const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);
+        return {
+            ok: true,
+            doc,
+            warnings
+        };
+    } catch (err) {
+        return {
+            ok: false,
+            error: serializeError(err)
+        };
+    }
+}

package/dist/esm/workers/parse.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task.\n// A Feature carries closures and cannot cross the thread boundary; its name can, and the\n// registry on this side resolves it back, which keeps the caller's selection intact.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[] } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    return { ok: true, doc, warnings };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["Tinypool","FEATURES","parseFile","featuresForFile","serializeError","cfg","featureNames","workerData","selected","filter","feature","includes","name","parseTask","file","doc","warnings","ok","err","error"],"mappings":"AAAA,OAAOA,cAAc,WAAW;AAEhC,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,eAAe,QAAQ,qBAAqB;AAErD,SAASC,cAAc,QAAQ,0BAA0B;AAgBzD,sCAAsC;AACtC,MAAM,EAAEC,GAAG,EAAEC,YAAY,EAAE,GAAGN,SAASO,UAAU;AACjD,4FAA4F;AAC5F,qDAAqD;AACrD,MAAMC,WAAWP,SAASQ,MAAM,CAAC,CAACC,UAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;AAEhF,eAAe,SAASC,UAAUC,IAAe;IAC/C,IAAI;QACF,MAAM,EAAEC,GAAG,EAAEC,QAAQ,EAAE,GAAGd,UAAUY,MAAMX,gBAAgBK,UAAUH,KAAKS,OAAOT;QAChF,OAAO;YAAEY,IAAI;YAAMF;YAAKC;QAAS;IACnC,EAAE,OAAOE,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOf,eAAec;QAAK;IACjD;AACF"}