sensemaking 0.18.0 → 0.18.2

package/dist/esm/store/index.js

@@ -18,10 +18,14 @@ const REGISTRY = {
         open: openDuckdb
     }
 };
-export async function openStore(cfg) {
-    const name = storeName(cfg);
+function entryFor(name) {
     const entry = REGISTRY[name];
     if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store "${name}"; available: ${Object.keys(REGISTRY).join(', ')}`);
+    return entry;
+}
+export async function openStore(cfg) {
+    const name = storeName(cfg);
+    const entry = entryFor(name);
     // The one unambiguous, config-level capability need: an embed block some preset actually
     // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level
     // (every preset defaults to wanting "words"), so a store lacking 'lexical' fails loudly at
@@ -31,5 +35,13 @@ export async function openStore(cfg) {
     }
     return entry.open(cfg);
 }
+// watch keeps its connection open for the run, so a file-locking store would fail every
+// other command on the tree; check before open, as openStore does for "vectors".
+export function requireWatchConcurrency(cfg) {
+    const name = storeName(cfg);
+    if (!entryFor(name).capabilities.has('watch-concurrency')) {
+        throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "watch-concurrency" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set "store" to "sqlite" in sense.config.json`);
+    }
+}
 export { getMeta, setMeta } from './meta.js';
 export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Capability } from './types.ts';\n\ntype StoreName = 'sqlite' | 'duckdb';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n};\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually\n  // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level\n  // (every preset defaults to wanting \"words\"), so a store lacking 'lexical' fails loudly at\n  // first use instead (see duckdb/store.ts's lexical.query()).\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\nexport { getMeta, setMeta } from './meta.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","openStore","cfg","name","entry","Object","keys","join","has","getMeta","setMeta","clearCache","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AAUxE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcH;QAAqBI,MAAML;IAAW;IAC9DM,QAAQ;QAAEF,cAAcL;QAAqBM,MAAMR;IAAW;AAChE;AAEA,OAAO,eAAeU,UAAUC,GAAmB;IACjD,MAAMC,OAAOd,UAAUa;IACvB,MAAME,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAId,WAAW,iBAAiB,CAAC,uBAAuB,EAAEa,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IAEnI,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,6DAA6D;IAC7D,IAAInB,gBAAgBc,QAAQ,CAACE,MAAMN,YAAY,CAACU,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAIlB,WAAW,4BAA4B,CAAC,OAAO,EAAEa,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACG;AACpB;AAEA,SAASO,OAAO,EAAEC,OAAO,QAAQ,YAAY;AAE7C,SAASC,UAAU,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Capability } from './types.ts';\n\ntype StoreName = 'sqlite' | 'duckdb';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually\n  // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level\n  // (every preset defaults to wanting \"words\"), so a store lacking 'lexical' fails loudly at\n  // first use instead (see duckdb/store.ts's lexical.query()).\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { getMeta, setMeta } from './meta.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","getMeta","setMeta","clearCache","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AAUxE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcH;QAAqBI,MAAML;IAAW;IAC9DM,QAAQ;QAAEF,cAAcL;QAAqBM,MAAMR;IAAW;AAChE;AAEA,SAASU,SAASC,IAAe;IAC/B,MAAMC,QAAQP,QAAQ,CAACM,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIb,WAAW,iBAAiB,CAAC,uBAAuB,EAAEY,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACT,UAAUU,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOb,UAAUmB;IACvB,MAAML,QAAQF,SAASC;IAEvB,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,6DAA6D;IAC7D,IAAId,gBAAgBoB,QAAQ,CAACL,MAAML,YAAY,CAACW,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAInB,WAAW,4BAA4B,CAAC,OAAO,EAAEY,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAMJ,IAAI,CAACS;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOb,UAAUmB;IACvB,IAAI,CAACP,SAASC,MAAMJ,YAAY,CAACW,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAInB,WAAW,4BAA4B,CAAC,OAAO,EAAEY,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,OAAO,EAAEC,OAAO,QAAQ,YAAY;AAE7C,SAASC,UAAU,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}

package/dist/esm/store/sqlite/reconcile.js

@@ -58,7 +58,7 @@ export async function reconcile(conn, cfg, baseDir) {
     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
     // reconciles stay silent (progress() has a threshold).
     const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
     report.finish();
     for (const col of newColumns)seenColumns.add(col);
     const allColumns = [

package/dist/esm/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on the frontmatter row it's coupled to, so every content\n    // delete/insert below must run after that row exists (vanished paths still have their\n    // frontmatter row at this point) and before it is removed (vanished frontmatter delete\n    // comes last).\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert into `content`, coupled to the frontmatter\n      // rowid (indexed via its PRIMARY KEY) rather than the UNINDEXED `path` column, which a\n      // per-row DELETE would otherwise scan the whole table to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaZ,aAAaW,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaZ,aAAaW,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaZ,aAAaW,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWrD,eAAe0B;IAChC,MAAM4B,cAAc,MAAM/C,WAAWkB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAAStD,SAAS,mBAAmB0C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAGrD,aAAasC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IAClHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIf,WACR,gBACA,CAAC,uBAAuB,EAAEgE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC7D,iBAAiBsC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACrB,YAAY0D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGxD,WAAWwD,GAAG,YAAY,EAAExD,WAAWwD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBAAgBc,MAAM;YAiEyCqD,iBAK7BA;QArEtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAEtE,WAAWoD,MAAM;QAErG,0FAA0F;QAC1F,sFAAsF;QACtF,uFAAuF;QACvF,eAAe;QACf,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,yFAAyF;YACzF,uFAAuF;YACvF,+DAA+D;YAC/D,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAP,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM+B,OAAOxB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIiE,OAAO;oBACxC,IAAIrB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAIkE,UAAU;oBACjD,QAAOlE,gBAAAA,IAAImE,IAAI,CAACvB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKuD,QAAQ,CAACd,WAAWe;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAM/D,aAAapB,gBAAgB4B,SAAS2D;YAC5C,MAAM5D,KAAKuD,QAAQ,CACjBjE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKuD,QAAQ,CACjB,4CACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAKuD,QAAQ,CACjB,6CACAM,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMtE,OAAOwC,WAAY,KAAK,MAAM+B,cAAcvE,IAAIwE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACzE,IAAIE,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE2E,WAAW7E,IAAI6E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM3F,QAAQiB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM1F,QAAQe,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASmD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG3C,IAAI,CAAC,OAAOwC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI3E,IAAI2E,IAAIH,KAAK,CAAC,KAAK/E,GAAG,CAAC,CAAC6E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAItG;IACpB,KAAK,MAAM,CAACuG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAEpE,GAAG,CAACuE,SAASC,KAAKF,QAAQrD,GAAG,CAACsD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAExE,GAAG,CAAC0E,MAAMD,QAAQrD,GAAG,CAACsD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAC/C,IAAMA,EAAEkC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQlE,IAAI,KAAK,IAAI,aAAa;WAAIkE;KAAQ,CAACrF,GAAG,CAAC+F,OAAO1D,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,eAAe2D,oBAAoBrG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMoG,OAAO,MAAMtG,KAAKQ,OAAO,CAAC;IAChC,MAAM+F,QAAQ,IAAInH,IAAI,AAAE,CAAA,MAAMkH,KAAK5F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ1B,UAAUwB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMiG,MAAMtF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAapB,gBAAgB4B,SAAS2D;IAC5C,MAAMjC,WAAqB,EAAE;IAC7B,MAAM6B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQrG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU8E,YAAY,EAAE,GAAG/H,UAAU8H;QAClD7E,SAASsC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC1E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBAAgBc,MAAM;QAC1B,IAAIwD,KAAK/B,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAACjE,oBAAoBkE;IAC/D;IACA,OAAO7B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on the frontmatter row it's coupled to, so every content\n    // delete/insert below must run after that row exists (vanished paths still have their\n    // frontmatter row at this point) and before it is removed (vanished frontmatter delete\n    // comes last).\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert into `content`, coupled to the frontmatter\n      // rowid (indexed via its PRIMARY KEY) rather than the UNINDEXED `path` column, which a\n      // per-row DELETE would otherwise scan the whole table to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaZ,aAAaW,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaZ,aAAaW,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaZ,aAAaW,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWrD,eAAe0B;IAChC,MAAM4B,cAAc,MAAM/C,WAAWkB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAAStD,SAAS,mBAAmB0C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMrD,aAAasC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIf,WACR,gBACA,CAAC,uBAAuB,EAAEgE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC7D,iBAAiBsC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACrB,YAAY0D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGxD,WAAWwD,GAAG,YAAY,EAAExD,WAAWwD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBAAgBc,MAAM;YAiEyCqD,iBAK7BA;QArEtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAEtE,WAAWoD,MAAM;QAErG,0FAA0F;QAC1F,sFAAsF;QACtF,uFAAuF;QACvF,eAAe;QACf,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,yFAAyF;YACzF,uFAAuF;YACvF,+DAA+D;YAC/D,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAP,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM+B,OAAOxB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIiE,OAAO;oBACxC,IAAIrB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAIkE,UAAU;oBACjD,QAAOlE,gBAAAA,IAAImE,IAAI,CAACvB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKuD,QAAQ,CAACd,WAAWe;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAM/D,aAAapB,gBAAgB4B,SAAS2D;YAC5C,MAAM5D,KAAKuD,QAAQ,CACjBjE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKuD,QAAQ,CACjB,4CACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAKuD,QAAQ,CACjB,6CACAM,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMtE,OAAOwC,WAAY,KAAK,MAAM+B,cAAcvE,IAAIwE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACzE,IAAIE,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE2E,WAAW7E,IAAI6E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM3F,QAAQiB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM1F,QAAQe,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASmD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG3C,IAAI,CAAC,OAAOwC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI3E,IAAI2E,IAAIH,KAAK,CAAC,KAAK/E,GAAG,CAAC,CAAC6E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAItG;IACpB,KAAK,MAAM,CAACuG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAEpE,GAAG,CAACuE,SAASC,KAAKF,QAAQrD,GAAG,CAACsD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAExE,GAAG,CAAC0E,MAAMD,QAAQrD,GAAG,CAACsD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAC/C,IAAMA,EAAEkC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQlE,IAAI,KAAK,IAAI,aAAa;WAAIkE;KAAQ,CAACrF,GAAG,CAAC+F,OAAO1D,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,eAAe2D,oBAAoBrG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMoG,OAAO,MAAMtG,KAAKQ,OAAO,CAAC;IAChC,MAAM+F,QAAQ,IAAInH,IAAI,AAAE,CAAA,MAAMkH,KAAK5F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ1B,UAAUwB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMiG,MAAMtF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAapB,gBAAgB4B,SAAS2D;IAC5C,MAAMjC,WAAqB,EAAE;IAC7B,MAAM6B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQrG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU8E,YAAY,EAAE,GAAG/H,UAAU8H;QAClD7E,SAASsC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC1E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBAAgBc,MAAM;QAC1B,IAAIwD,KAAK/B,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAACjE,oBAAoBkE;IAC/D;IACA,OAAO7B;AACT"}

package/dist/esm/watch.js

@@ -2,7 +2,7 @@ import { watch as fsWatch } from 'node:fs';
 import { STATE_DIR } from './config/index.js';
 import { SenseError } from './errors.js';
 import { guardedTick } from './lib/guarded-tick.js';
-import { docCount, getMeta, openStore, setMeta } from './store/index.js';
+import { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.js';
 // Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.
 const DEBOUNCE_MS = 200;
 const HEARTBEAT_INTERVAL_MS = 5000;
@@ -10,6 +10,7 @@ const STALE_HEARTBEAT_MS = 15000;
 // Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.
 export async function runWatch(cfg, opts = {}) {
     var _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs;
+    requireWatchConcurrency(cfg);
     const onEvent = (_opts_onEvent = opts.onEvent) !== null && _opts_onEvent !== void 0 ? _opts_onEvent : ()=>{};
     const debounceMs = (_opts_debounceMs = opts.debounceMs) !== null && _opts_debounceMs !== void 0 ? _opts_debounceMs : DEBOUNCE_MS;
     const heartbeatIntervalMs = (_opts_heartbeatIntervalMs = opts.heartbeatIntervalMs) !== null && _opts_heartbeatIntervalMs !== void 0 ? _opts_heartbeatIntervalMs : HEARTBEAT_INTERVAL_MS;
@@ -43,27 +44,38 @@ export async function runWatch(cfg, opts = {}) {
     };
     await touchHeartbeat();
     let debounceTimer = null;
+    // A reconcile that has already started owns the store, and on a bulk reparse a live worker
+    // pool as well. Shutdown waits on this rather than closing the connection underneath it.
+    let inFlight = null;
     const scheduleReconcile = ()=>{
         if (debounceTimer) clearTimeout(debounceTimer);
-        debounceTimer = setTimeout(async ()=>{
+        debounceTimer = setTimeout(()=>{
             debounceTimer = null;
-            try {
-                const { parsed, warnings } = await store.reconcile();
-                onEvent({
-                    type: 'reconciled',
-                    parsed,
-                    total: await docCount(store),
-                    warnings
-                });
-            } catch (err) {
-                onEvent({
-                    type: 'reconcile-error',
-                    message: err.message
-                });
-            }
+            inFlight = (async ()=>{
+                try {
+                    const { parsed, warnings } = await store.reconcile();
+                    onEvent({
+                        type: 'reconciled',
+                        parsed,
+                        total: await docCount(store),
+                        warnings
+                    });
+                } catch (err) {
+                    onEvent({
+                        type: 'reconcile-error',
+                        message: err.message
+                    });
+                } finally{
+                    inFlight = null;
+                }
+            })();
         }, debounceMs);
     };
-    // Ignore our own state dir, or the heartbeat write would retrigger itself forever.
+    // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event
+    // whose filename the platform could not resolve (null, which fs.watch does deliver under
+    // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs
+    // less than missing a real edit. That is why the guard cannot promise zero reconciles, only
+    // that an identified state-dir write is never one of them.
     const watcher = fsWatch(baseDir, {
         recursive: true
     }, (_event, filename)=>{
@@ -85,6 +97,7 @@ export async function runWatch(cfg, opts = {}) {
             clearInterval(heartbeatTimer);
             if (debounceTimer) clearTimeout(debounceTimer);
             watcher.close();
+            await inFlight;
             await setMeta(store, 'watch_heartbeat', null);
             await setMeta(store, 'watch_pid', null);
             await store.close();

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(async () => {\n      debounceTimer = null;\n      try {\n        const { parsed, warnings } = await store.reconcile();\n        onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,mBAAmB;AAEzE,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QACzDA,eACGA,kBACSA;IAF5B,MAAMC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMhB,UAAUM;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMnB,QAAQY,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI5B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE6B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAM/B,SAASa;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,MAAMC,oBAAoB;QACxB,IAAID,eAAeE,aAAaF;QAChCA,gBAAgBG,WAAW;YACzBH,gBAAgB;YAChB,IAAI;gBACF,MAAM,EAAErB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM6B,SAAS;gBAClDhC,QAAQ;oBAAEmB,MAAM;oBAAcZ;oBAAQc,OAAO,MAAM/B,SAASa;oBAAQE;gBAAS;YAC/E,EAAE,OAAO4B,KAAK;gBACZjC,QAAQ;oBAAEmB,MAAM;oBAAmBe,SAAS,AAACD,IAAcC,OAAO;gBAAC;YACrE;QACF,GAAGjC;IACL;IAEA,mFAAmF;IACnF,MAAMkC,UAAUjD,QAAQuB,SAAS;QAAE2B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACpD,YAAY;QACpE0C;IACF;IACA,MAAMW,iBAAiBC,YACrBpD,YAAYkC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIwC,QAAc,CAACC;YAmBpB5C,cACCA;QAnBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM6C,WAAW;gBAKf7C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQmB,GAAG,CAAC,UAAUD;YACtBlB,QAAQmB,GAAG,CAAC,WAAWD;aACvB7C,eAAAA,KAAK+C,MAAM,cAAX/C,mCAAAA,aAAagD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIZ,eAAeE,aAAaF;YAChCO,QAAQnB,KAAK;YACb,MAAMvB,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB2B;QACF;QACAjB,QAAQuB,IAAI,CAAC,UAAUL;QACvBlB,QAAQuB,IAAI,CAAC,WAAWL;QACxB,KAAI7C,eAAAA,KAAK+C,MAAM,cAAX/C,mCAAAA,aAAamD,OAAO,EAAEN;cACrB7C,gBAAAA,KAAK+C,MAAM,cAAX/C,oCAAAA,cAAaoD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event\n  // whose filename the platform could not resolve (null, which fs.watch does deliver under\n  // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs\n  // less than missing a real edit. That is why the guard cannot promise zero reconciles, only\n  // that an identified state-dir write is never one of them.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","inFlight","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAElG,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUO;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMpB,QAAQa,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMhC,SAASc;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAIC,WAAiC;IACrC,MAAMC,oBAAoB;QACxB,IAAIF,eAAeG,aAAaH;QAChCA,gBAAgBI,WAAW;YACzBJ,gBAAgB;YAChBC,WAAW,AAAC,CAAA;gBACV,IAAI;oBACF,MAAM,EAAEtB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM8B,SAAS;oBAClDjC,QAAQ;wBAAEmB,MAAM;wBAAcZ;wBAAQc,OAAO,MAAMhC,SAASc;wBAAQE;oBAAS;gBAC/E,EAAE,OAAO6B,KAAK;oBACZlC,QAAQ;wBAAEmB,MAAM;wBAAmBgB,SAAS,AAACD,IAAcC,OAAO;oBAAC;gBACrE,SAAU;oBACRN,WAAW;gBACb;YACF,CAAA;QACF,GAAG5B;IACL;IAEA,4FAA4F;IAC5F,yFAAyF;IACzF,4FAA4F;IAC5F,4FAA4F;IAC5F,2DAA2D;IAC3D,MAAMmC,UAAUnD,QAAQwB,SAAS;QAAE4B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACtD,YAAY;QACpE4C;IACF;IACA,MAAMW,iBAAiBC,YACrBtD,YAAYmC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIyC,QAAc,CAACC;YAoBpB7C,cACCA;QApBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM8C,WAAW;gBAKf9C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQoB,GAAG,CAAC,UAAUD;YACtBnB,QAAQoB,GAAG,CAAC,WAAWD;aACvB9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaiD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIb,eAAeG,aAAaH;YAChCQ,QAAQpB,KAAK;YACb,MAAMa;YACN,MAAMpC,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB4B;QACF;QACAlB,QAAQwB,IAAI,CAAC,UAAUL;QACvBnB,QAAQwB,IAAI,CAAC,WAAWL;QACxB,KAAI9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaoD,OAAO,EAAEN;cACrB9C,gBAAAA,KAAKgD,MAAM,cAAXhD,oCAAAA,cAAaqD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}

package/dist/esm/workers/parse.d.ts

@@ -0,0 +1,18 @@
+import type { Config, FeatureName } from '../config/index.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from '../scan/list.js';
+import type { WorkerErrorPayload } from '../scan/worker-error.js';
+export interface ParseWorkerData {
+    cfg: Config;
+    featureNames: FeatureName[];
+}
+export type ParseTask = FileStat;
+export type ParseTaskResult = {
+    ok: true;
+    doc: ParsedDoc;
+    warnings: string[];
+} | {
+    ok: false;
+    error: WorkerErrorPayload;
+};
+export default function parseTask(file: ParseTask): ParseTaskResult;

package/dist/esm/workers/parse.js

@@ -0,0 +1,25 @@
+import Tinypool from 'tinypool';
+import { FEATURES } from '../features/index.js';
+import { parseFile } from '../scan/index.js';
+import { featuresForFile } from '../scan/reparse.js';
+import { serializeError } from '../scan/worker-error.js';
+// Read once per worker, not per task.
+const { cfg, featureNames } = Tinypool.workerData;
+// Filtering the registry (rather than mapping the names) keeps registry order, which is the
+// order `extracted` keys land in on the serial path.
+const selected = FEATURES.filter((feature)=>featureNames.includes(feature.name));
+export default function parseTask(file) {
+    try {
+        const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);
+        return {
+            ok: true,
+            doc,
+            warnings
+        };
+    } catch (err) {
+        return {
+            ok: false,
+            error: serializeError(err)
+        };
+    }
+}

package/dist/esm/workers/parse.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task.\n// A Feature carries closures and cannot cross the thread boundary; its name can, and the\n// registry on this side resolves it back, which keeps the caller's selection intact.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[] } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    return { ok: true, doc, warnings };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["Tinypool","FEATURES","parseFile","featuresForFile","serializeError","cfg","featureNames","workerData","selected","filter","feature","includes","name","parseTask","file","doc","warnings","ok","err","error"],"mappings":"AAAA,OAAOA,cAAc,WAAW;AAEhC,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,eAAe,QAAQ,qBAAqB;AAErD,SAASC,cAAc,QAAQ,0BAA0B;AAgBzD,sCAAsC;AACtC,MAAM,EAAEC,GAAG,EAAEC,YAAY,EAAE,GAAGN,SAASO,UAAU;AACjD,4FAA4F;AAC5F,qDAAqD;AACrD,MAAMC,WAAWP,SAASQ,MAAM,CAAC,CAACC,UAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;AAEhF,eAAe,SAASC,UAAUC,IAAe;IAC/C,IAAI;QACF,MAAM,EAAEC,GAAG,EAAEC,QAAQ,EAAE,GAAGd,UAAUY,MAAMX,gBAAgBK,UAAUH,KAAKS,OAAOT;QAChF,OAAO;YAAEY,IAAI;YAAMF;YAAKC;QAAS;IACnC,EAAE,OAAOE,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOf,eAAec;QAAK;IACjD;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.18.0",
+  "version": "0.18.2",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -92,6 +92,7 @@
     "micromark-extension-gfm-strikethrough": "^2.0.0",
     "micromark-extension-gfm-table": "^2.0.0",
     "micromark-extension-gfm-task-list-item": "^2.0.0",
+    "tinypool": "^2.1.2",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
@@ -100,6 +101,7 @@
     "@types/mocha": "*",
     "@types/node": "*",
     "cr": "^0.1.0",
+    "fs-remove-compat": "^1.0.4",
     "node-version-use": "*",
     "ts-dev-stack": "*",
     "tsds-config": "*"

package/skills/sense-setup/SKILL.md

@@ -10,7 +10,7 @@ Querying an existing tree is the `sense` skill. This one covers making a tree: i
 ## Setup
 
 - `npm install -g sensemaking`, then `sense init` at the tree root writes `sense.config.json`: two presets (`default`, and `large` showing what a big tree tunes) and an `embed` block naming the model. The model fetches once per machine at the first vector search (progress on stderr); `sense download` prefetches it instead where that timing matters (CI, air-gapped setup). Config discovery walks up from cwd; `--config <path>` overrides.
-- **Backing store.** The config's `store` key: `sqlite` (default, zero-dependency, Node's built-in SQLite) or `duckdb` (experimental; the first command that opens a duckdb tree installs `@duckdb/node-api` on its own, a one-time native download of about 110 MB). The same commands, tables, and `has`/`basename`/`segment` functions run on both. The difference is FTS5: under `duckdb`, `search` text and raw `MATCH` reject FTS5's prefix, boolean, `NEAR`, initial-token, and column-filter operators with a named error, and sqlite's FTS5 SQL (`MATCH`, `snippet()`, `bm25()`) does not run, so saved queries written in that syntax are sqlite dialect. A tree whose saved queries or search vocabulary depend on FTS5 operators is a tree that stays on `sqlite`. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`); switching stores is a rebuild, not a migration.
+- **Backing store.** The config's `store` key: `sqlite` (default, zero-dependency, Node's built-in SQLite) or `duckdb` (experimental; the first command that opens a duckdb tree installs `@duckdb/node-api` on its own, a one-time native download of about 110 MB). The same commands, tables, and `has`/`basename`/`segment` functions run on both. The difference is FTS5: under `duckdb`, `search` text and raw `MATCH` reject FTS5's prefix, boolean, `NEAR`, initial-token, and column-filter operators with a named error, and sqlite's FTS5 SQL (`MATCH`, `snippet()`, `bm25()`) does not run, so saved queries written in that syntax are sqlite dialect. A tree whose saved queries or search vocabulary depend on FTS5 operators is a tree that stays on `sqlite`. `sense watch` is the other store-bound command: under `duckdb` it is a named error, because the store locks its cache file for the watcher's connection, which would block every other command on the tree for the watch's lifetime. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`); switching stores is a rebuild, not a migration.
 - Globs resolve relative to the config file, never the cwd.
 - `sense status` and `sense map` show each preset's coverage (files matched, embedded count), so what a config actually indexes is always visible in output. A config edit that changes coverage rebuilds the cache and names the preset that caused it on stderr.