sensemaking 0.18.1 → 0.18.2

package/dist/esm/scan/reparse.js

@@ -1,18 +1,118 @@
+import { createRequire } from 'node:module';
+import { availableParallelism } from 'node:os';
+import { dirname, join } from 'node:path';
 import { parseFile } from './index.js';
+import { reviveError } from './worker-error.js';
+// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house
+// deferral reaches it. Deferred (not a top-level import) because most reconciles stay below
+// DEFAULT_THRESHOLD and should not pay for constructing pool machinery they never use.
+const _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;
+// Measured 2026-08-29 on a 14-logical-core Apple M4 Pro: serial and pooled cross between 160
+// and 180 files (pool fixed cost here is ~85-100ms, not the ~300ms a heavier module graph
+// costs elsewhere), so 200 keeps a safety margin without giving up real wins. 8 workers matches
+// an interleaved worker-count sweep at 6,566 files -- 8 and 10 tie within noise, 12+ trends
+// worse (main-thread + OS contention on the 6 non-parse-worker cores).
+const DEFAULT_THRESHOLD = 200;
+const DEFAULT_MAX_WORKERS = 8;
+// Worker MUST always load from dist/cjs/ (house pattern: install-module-linked,
+// install-optional, node-version-install, node-exec-path all carry this same rule). A
+// worker_threads thread is a fresh realm and does not inherit the main thread's TS loader
+// hook, so a source path cannot work on any platform; dist/cjs loads from the esm and cjs
+// builds alike. Root is found the way cli.ts finds it, so this also resolves when the suite
+// runs against src/ (which already requires a build for the exports specs).
+// Resolved on first pooled dispatch, not at import: a tree under the threshold never pools and
+// must not pay for this, let alone fail on it.
+let workerFile;
+function resolveWorkerFile() {
+    if (workerFile) return workerFile;
+    const load = createRequire(import.meta.url);
+    for (const rel of [
+        '..',
+        '../..',
+        '../../..'
+    ]){
+        try {
+            if (load(`${rel}/package.json`).name === 'sensemaking') {
+                workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');
+                return workerFile;
+            }
+        } catch  {}
+    }
+    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
+}
+// Per-file feature filtering, shared by the serial loop (which reads `features` straight from
+// the caller, real registry or a test double) and the worker path (which can only pass `cfg`
+// across the thread boundary, so it derives `features` from activeFeatures(cfg) itself).
+export function featuresForFile(features, cfg, file) {
+    return features.filter((feature)=>!feature.enabledForFile || feature.enabledForFile(cfg, file));
+}
+function reparseSerial(files, features, cfg, onParsed) {
+    const results = [];
+    let done = 0;
+    for (const file of files){
+        const fileFeatures = featuresForFile(features, cfg, file);
+        results.push(parseFile(file, fileFeatures, cfg));
+        onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
+    }
+    return results;
+}
+// Never the tree: a worker task carries one FileStat and returns only what parseFile already
+// returns -- extracted text and feature values, never an mdast tree. The worker reads the file
+// itself. Pool created once here and destroyed once dispatch finishes; never per task.
+async function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
+    const { Tinypool } = _require('tinypool');
+    // cfg and the caller's feature selection are constant for the whole dispatch, so they cross
+    // once per worker as workerData rather than once per file in the task payload. Features carry
+    // closures and cannot cross at all; their names can, and the worker resolves them back
+    // against the same registry, so a caller passing a subset gets that subset in both modes.
+    const workerData = {
+        cfg,
+        featureNames: features.map((feature)=>feature.name)
+    };
+    const pool = new Tinypool({
+        filename: resolveWorkerFile(),
+        minThreads: maxWorkers,
+        maxThreads: maxWorkers,
+        workerData
+    });
+    let done = 0;
+    try {
+        // Promise.all over a mapped array is load-bearing: the resolved array keeps `files`
+        // order regardless of which task finishes first, which is what first-seen column order
+        // and warning order depend on downstream. Progress ticks fire in completion order instead
+        // (only the count matters for a progress bar) -- free, since it rides the same per-task
+        // response tinypool already sends, no extra ping.
+        const results = await Promise.all(files.map(async (file)=>{
+            const result = await pool.run(file);
+            if (!result.ok) throw reviveError(result.error);
+            onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
+            return {
+                doc: result.doc,
+                warnings: result.warnings
+            };
+        }));
+        await pool.destroy();
+        return results;
+    } catch (err) {
+        // Tear the pool down before rethrowing, but never let a destroy failure replace the parse
+        // failure that caused it: that one names the file, this one names nothing. Not a `finally`
+        // for exactly that reason.
+        await pool.destroy().catch(()=>{});
+        throw err;
+    }
+}
 // `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick
 // call; pass one in to keep progress reporting working without this module owning a reporter.
-export function reparseFiles(files, features, cfg, knownColumns, onParsed) {
+export async function reparseFiles(files, features, cfg, knownColumns, onParsed, options = {}) {
+    var _options_threshold, _options_maxWorkers;
+    const threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold : DEFAULT_THRESHOLD;
+    const maxWorkers = (_options_maxWorkers = options.maxWorkers) !== null && _options_maxWorkers !== void 0 ? _options_maxWorkers : Math.min(DEFAULT_MAX_WORKERS, availableParallelism());
+    const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);
     const docs = [];
     const warnings = [];
     const newColumns = [];
     const seen = new Set(knownColumns);
-    let done = 0;
-    for (const file of files){
-        // A doc only gets extract/store from features that apply to it (currently: embed, via
-        // FileStat.embed -- true iff the config names an embedding model).
-        const fileFeatures = features.filter((feature)=>!feature.enabledForFile || feature.enabledForFile(cfg, file));
-        const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures, cfg);
-        onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
+    for (const { doc, warnings: fileWarnings } of results){
         warnings.push(...fileWarnings);
         for (const key of Object.keys(doc.data)){
             if (!seen.has(key)) {

package/dist/esm/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving by\n// construction (one pass over `files`, pushed in order) so a future concurrent dispatch can\n// replace the loop body without changing this contract or either call site.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order across `files` -- the order\n  // callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void): ReparseResult {\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  let done = 0;\n\n  for (const file of files) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures, cfg);\n    onParsed?.(++done);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["parseFile","reparseFiles","files","features","cfg","knownColumns","onParsed","docs","warnings","newColumns","seen","Set","done","file","fileFeatures","filter","feature","enabledForFile","doc","fileWarnings","push","key","Object","keys","data","has","add"],"mappings":"AAGA,SAASA,SAAS,QAAQ,aAAa;AAcvC,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,aAAaC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,YAAiC,EAAEC,QAAiC;IACpJ,MAAMC,OAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIN;IACrB,IAAIO,OAAO;IAEX,KAAK,MAAMC,QAAQX,MAAO;QACxB,sFAAsF;QACtF,mEAAmE;QACnE,MAAMY,eAAeX,SAASY,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACb,KAAKS;QACzG,MAAM,EAAEK,GAAG,EAAEV,UAAUW,YAAY,EAAE,GAAGnB,UAAUa,MAAMC,cAAcV;QACtEE,qBAAAA,+BAAAA,SAAW,EAAEM;QACbJ,SAASY,IAAI,IAAID;QACjB,KAAK,MAAME,OAAOC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACd,KAAKe,GAAG,CAACJ,MAAM;gBAClBX,KAAKgB,GAAG,CAACL;gBACTZ,WAAWW,IAAI,CAACC;YAClB;QACF;QACAd,KAAKa,IAAI,CAACF;IACZ;IAEA,OAAO;QAAEX;QAAMC;QAAUC;IAAW;AACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house\n// deferral reaches it. Deferred (not a top-level import) because most reconciles stay below\n// DEFAULT_THRESHOLD and should not pay for constructing pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving by\n// construction (one pass over `files`, pushed in order) so a future concurrent dispatch can\n// replace the loop body without changing this contract or either call site.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order across `files` -- the order\n  // callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-logical-core Apple M4 Pro: serial and pooled cross between 160\n// and 180 files (pool fixed cost here is ~85-100ms, not the ~300ms a heavier module graph\n// costs elsewhere), so 200 keeps a safety margin without giving up real wins. 8 workers matches\n// an interleaved worker-count sweep at 6,566 files -- 8 and 10 tie within noise, 12+ trends\n// worse (main-thread + OS contention on the 6 non-parse-worker cores).\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST always load from dist/cjs/ (house pattern: install-module-linked,\n// install-optional, node-version-install, node-exec-path all carry this same rule). A\n// worker_threads thread is a fresh realm and does not inherit the main thread's TS loader\n// hook, so a source path cannot work on any platform; dist/cjs loads from the esm and cjs\n// builds alike. Root is found the way cli.ts finds it, so this also resolves when the suite\n// runs against src/ (which already requires a build for the exports specs).\n// Resolved on first pooled dispatch, not at import: a tree under the threshold never pools and\n// must not pay for this, let alone fail on it.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (which reads `features` straight from\n// the caller, real registry or a test double) and the worker path (which can only pass `cfg`\n// across the thread boundary, so it derives `features` from activeFeatures(cfg) itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile already\n// returns -- extracted text and feature values, never an mdast tree. The worker reads the file\n// itself. Pool created once here and destroyed once dispatch finishes; never per task.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the caller's feature selection are constant for the whole dispatch, so they cross\n  // once per worker as workerData rather than once per file in the task payload. Features carry\n  // closures and cannot cross at all; their names can, and the worker resolves them back\n  // against the same registry, so a caller passing a subset gets that subset in both modes.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files`\n    // order regardless of which task finishes first, which is what first-seen column order\n    // and warning order depend on downstream. Progress ticks fire in completion order instead\n    // (only the count matters for a progress bar) -- free, since it rides the same per-task\n    // response tinypool already sends, no extra ping.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but never let a destroy failure replace the parse\n    // failure that caused it: that one names the file, this one names nothing. Not a `finally`\n    // for exactly that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["createRequire","availableParallelism","dirname","join","parseFile","reviveError","_require","require","url","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","Tinypool","workerData","featureNames","map","pool","filename","minThreads","maxThreads","Promise","all","result","run","ok","error","doc","warnings","destroy","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","newColumns","seen","Set","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,oBAAoB,QAAQ,UAAU;AAC/C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAQ1C,SAASC,SAAS,QAAQ,aAAa;AAEvC,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,yFAAyF;AACzF,4FAA4F;AAC5F,uFAAuF;AACvF,MAAMC,WAAW,OAAOC,YAAY,cAAcP,cAAc,YAAYQ,GAAG,IAAID;AAqBnF,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,4FAA4F;AAC5F,uEAAuE;AACvE,MAAME,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,gFAAgF;AAChF,sFAAsF;AACtF,0FAA0F;AAC1F,0FAA0F;AAC1F,4FAA4F;AAC5F,4EAA4E;AAC5E,+FAA+F;AAC/F,+CAA+C;AAC/C,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOb,cAAc,YAAYQ,GAAG;IAC1C,KAAK,MAAMM,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaR,KAAKD,QAAQW,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAIA,8FAA8F;AAC9F,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAAC3B,UAAUiB,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,uFAAuF;AACvF,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB;IAClJ,MAAM,EAAEC,QAAQ,EAAE,GAAG5B,SAAS;IAC9B,4FAA4F;IAC5F,8FAA8F;IAC9F,uFAAuF;IACvF,0FAA0F;IAC1F,MAAM6B,aAA8B;QAAEf;QAAKgB,cAAcjB,SAASkB,GAAG,CAAC,CAACd,UAAYA,QAAQR,IAAI;IAAE;IACjG,MAAMuB,OAAO,IAAIJ,SAAS;QAAEK,UAAU3B;QAAqB4B,YAAYP;QAAYQ,YAAYR;QAAYE;IAAW;IACtH,IAAIN,OAAO;IACX,IAAI;QACF,oFAAoF;QACpF,uFAAuF;QACvF,0FAA0F;QAC1F,wFAAwF;QACxF,kDAAkD;QAClD,MAAMD,UAAU,MAAMc,QAAQC,GAAG,CAC/BjB,MAAMW,GAAG,CAAC,OAAOhB;YACf,MAAMuB,SAAU,MAAMN,KAAKO,GAAG,CAACxB;YAC/B,IAAI,CAACuB,OAAOE,EAAE,EAAE,MAAMzC,YAAYuC,OAAOG,KAAK;YAC9CpB,qBAAAA,+BAAAA,SAAW,EAAEE;YACb,OAAO;gBAAEmB,KAAKJ,OAAOI,GAAG;gBAAEC,UAAUL,OAAOK,QAAQ;YAAC;QACtD;QAEF,MAAMX,KAAKY,OAAO;QAClB,OAAOtB;IACT,EAAE,OAAOuB,KAAK;QACZ,0FAA0F;QAC1F,2FAA2F;QAC3F,2BAA2B;QAC3B,MAAMb,KAAKY,OAAO,GAAGE,KAAK,CAAC,KAAO;QAClC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAa3B,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEkC,YAAiC,EAAE3B,QAAiC,EAAE4B,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB9C;IACvC,MAAMwB,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAChD,qBAAqBT;IACvE,MAAM2B,UAAUF,MAAMiC,MAAM,IAAIH,YAAY,MAAMxB,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,cAAcR,cAAcC,OAAOP,UAAUC,KAAKO;IAElJ,MAAMiC,OAAoB,EAAE;IAC5B,MAAMX,WAAqB,EAAE;IAC7B,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIT;IACrB,KAAK,MAAM,EAAEN,GAAG,EAAEC,UAAUe,YAAY,EAAE,IAAIpC,QAAS;QACrDqB,SAASlB,IAAI,IAAIiC;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACnB,IAAIoB,IAAI,EAAG;YACvC,IAAI,CAACN,KAAKO,GAAG,CAACJ,MAAM;gBAClBH,KAAKQ,GAAG,CAACL;gBACTJ,WAAW9B,IAAI,CAACkC;YAClB;QACF;QACAL,KAAK7B,IAAI,CAACiB;IACZ;IAEA,OAAO;QAAEY;QAAMX;QAAUY;IAAW;AACtC"}

package/dist/esm/scan/worker-error.d.ts

@@ -0,0 +1,9 @@
+import { type SenseErrorCode } from '../errors.js';
+export interface WorkerErrorPayload {
+    name: string;
+    code?: SenseErrorCode;
+    message: string;
+    stack?: string;
+}
+export declare function serializeError(err: unknown): WorkerErrorPayload;
+export declare function reviveError(payload: WorkerErrorPayload): Error;

package/dist/esm/scan/worker-error.js

@@ -0,0 +1,19 @@
+import { SenseError } from '../errors.js';
+export function serializeError(err) {
+    if (err instanceof Error) return {
+        name: err.name,
+        code: err instanceof SenseError ? err.code : undefined,
+        message: err.message,
+        stack: err.stack
+    };
+    return {
+        name: 'Error',
+        message: String(err)
+    };
+}
+export function reviveError(payload) {
+    const err = payload.code ? new SenseError(payload.code, payload.message) : new Error(payload.message);
+    err.name = payload.name;
+    if (payload.stack) err.stack = payload.stack;
+    return err;
+}

package/dist/esm/scan/worker-error.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/worker-error.ts"],"sourcesContent":["import { SenseError, type SenseErrorCode } from '../errors.ts';\n\n// Structured clone across a worker_threads boundary drops a subclass and its custom\n// properties, so a SenseError arrives as a plain Error with `code` undefined -- and `code`\n// drives caller behavior. The worker serializes explicitly (computed on its side, where\n// `instanceof SenseError` is still meaningful) and the main thread rebuilds by hand.\nexport interface WorkerErrorPayload {\n  name: string;\n  code?: SenseErrorCode;\n  message: string;\n  stack?: string;\n}\n\nexport function serializeError(err: unknown): WorkerErrorPayload {\n  if (err instanceof Error) return { name: err.name, code: err instanceof SenseError ? err.code : undefined, message: err.message, stack: err.stack };\n  return { name: 'Error', message: String(err) };\n}\n\nexport function reviveError(payload: WorkerErrorPayload): Error {\n  const err = payload.code ? new SenseError(payload.code, payload.message) : new Error(payload.message);\n  err.name = payload.name;\n  if (payload.stack) err.stack = payload.stack;\n  return err;\n}\n"],"names":["SenseError","serializeError","err","Error","name","code","undefined","message","stack","String","reviveError","payload"],"mappings":"AAAA,SAASA,UAAU,QAA6B,eAAe;AAa/D,OAAO,SAASC,eAAeC,GAAY;IACzC,IAAIA,eAAeC,OAAO,OAAO;QAAEC,MAAMF,IAAIE,IAAI;QAAEC,MAAMH,eAAeF,aAAaE,IAAIG,IAAI,GAAGC;QAAWC,SAASL,IAAIK,OAAO;QAAEC,OAAON,IAAIM,KAAK;IAAC;IAClJ,OAAO;QAAEJ,MAAM;QAASG,SAASE,OAAOP;IAAK;AAC/C;AAEA,OAAO,SAASQ,YAAYC,OAA2B;IACrD,MAAMT,MAAMS,QAAQN,IAAI,GAAG,IAAIL,WAAWW,QAAQN,IAAI,EAAEM,QAAQJ,OAAO,IAAI,IAAIJ,MAAMQ,QAAQJ,OAAO;IACpGL,IAAIE,IAAI,GAAGO,QAAQP,IAAI;IACvB,IAAIO,QAAQH,KAAK,EAAEN,IAAIM,KAAK,GAAGG,QAAQH,KAAK;IAC5C,OAAON;AACT"}

package/dist/esm/store/duckdb/reconcile.js

@@ -56,7 +56,7 @@ export async function reconcile(conn, cfg, baseDir) {
     const features = activeFeatures(cfg);
     const seenColumns = await getColumns(conn);
     const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
     report.finish();
     for (const col of newColumns)seenColumns.add(col);
     const allColumns = [

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per\n// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain\n// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from\n// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one\n// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared\n// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the\n// only DuckDB type that can hold the different JS types mapValue() produces across files for\n// the same key).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity\n// fence anyway so a runaway frontmatter generator fails with a clear message instead of an\n// unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,4FAA4F;AAC5F,8FAA8F;AAC9F,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iBAAiB;AACjB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,6FAA6F;AAC7F,2FAA2F;AAC3F,8BAA8B;AAC9B,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAGjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IAClHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per\n// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain\n// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from\n// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one\n// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared\n// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the\n// only DuckDB type that can hold the different JS types mapValue() produces across files for\n// the same key).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity\n// fence anyway so a runaway frontmatter generator fails with a clear message instead of an\n// unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,4FAA4F;AAC5F,8FAA8F;AAC9F,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iBAAiB;AACjB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,6FAA6F;AAC7F,2FAA2F;AAC3F,8BAA8B;AAC9B,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/sqlite/reconcile.js

@@ -58,7 +58,7 @@ export async function reconcile(conn, cfg, baseDir) {
     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
     // reconciles stay silent (progress() has a threshold).
     const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
     report.finish();
     for (const col of newColumns)seenColumns.add(col);
     const allColumns = [

package/dist/esm/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on the frontmatter row it's coupled to, so every content\n    // delete/insert below must run after that row exists (vanished paths still have their\n    // frontmatter row at this point) and before it is removed (vanished frontmatter delete\n    // comes last).\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert into `content`, coupled to the frontmatter\n      // rowid (indexed via its PRIMARY KEY) rather than the UNINDEXED `path` column, which a\n      // per-row DELETE would otherwise scan the whole table to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaZ,aAAaW,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaZ,aAAaW,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaZ,aAAaW,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWrD,eAAe0B;IAChC,MAAM4B,cAAc,MAAM/C,WAAWkB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAAStD,SAAS,mBAAmB0C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAGrD,aAAasC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IAClHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIf,WACR,gBACA,CAAC,uBAAuB,EAAEgE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC7D,iBAAiBsC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACrB,YAAY0D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGxD,WAAWwD,GAAG,YAAY,EAAExD,WAAWwD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBAAgBc,MAAM;YAiEyCqD,iBAK7BA;QArEtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAEtE,WAAWoD,MAAM;QAErG,0FAA0F;QAC1F,sFAAsF;QACtF,uFAAuF;QACvF,eAAe;QACf,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,yFAAyF;YACzF,uFAAuF;YACvF,+DAA+D;YAC/D,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAP,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM+B,OAAOxB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIiE,OAAO;oBACxC,IAAIrB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAIkE,UAAU;oBACjD,QAAOlE,gBAAAA,IAAImE,IAAI,CAACvB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKuD,QAAQ,CAACd,WAAWe;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAM/D,aAAapB,gBAAgB4B,SAAS2D;YAC5C,MAAM5D,KAAKuD,QAAQ,CACjBjE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKuD,QAAQ,CACjB,4CACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAKuD,QAAQ,CACjB,6CACAM,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMtE,OAAOwC,WAAY,KAAK,MAAM+B,cAAcvE,IAAIwE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACzE,IAAIE,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE2E,WAAW7E,IAAI6E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM3F,QAAQiB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM1F,QAAQe,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASmD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG3C,IAAI,CAAC,OAAOwC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI3E,IAAI2E,IAAIH,KAAK,CAAC,KAAK/E,GAAG,CAAC,CAAC6E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAItG;IACpB,KAAK,MAAM,CAACuG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAEpE,GAAG,CAACuE,SAASC,KAAKF,QAAQrD,GAAG,CAACsD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAExE,GAAG,CAAC0E,MAAMD,QAAQrD,GAAG,CAACsD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAC/C,IAAMA,EAAEkC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQlE,IAAI,KAAK,IAAI,aAAa;WAAIkE;KAAQ,CAACrF,GAAG,CAAC+F,OAAO1D,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,eAAe2D,oBAAoBrG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMoG,OAAO,MAAMtG,KAAKQ,OAAO,CAAC;IAChC,MAAM+F,QAAQ,IAAInH,IAAI,AAAE,CAAA,MAAMkH,KAAK5F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ1B,UAAUwB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMiG,MAAMtF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAapB,gBAAgB4B,SAAS2D;IAC5C,MAAMjC,WAAqB,EAAE;IAC7B,MAAM6B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQrG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU8E,YAAY,EAAE,GAAG/H,UAAU8H;QAClD7E,SAASsC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC1E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBAAgBc,MAAM;QAC1B,IAAIwD,KAAK/B,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAACjE,oBAAoBkE;IAC/D;IACA,OAAO7B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on the frontmatter row it's coupled to, so every content\n    // delete/insert below must run after that row exists (vanished paths still have their\n    // frontmatter row at this point) and before it is removed (vanished frontmatter delete\n    // comes last).\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert into `content`, coupled to the frontmatter\n      // rowid (indexed via its PRIMARY KEY) rather than the UNINDEXED `path` column, which a\n      // per-row DELETE would otherwise scan the whole table to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaZ,aAAaW,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaZ,aAAaW,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaZ,aAAaW,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWrD,eAAe0B;IAChC,MAAM4B,cAAc,MAAM/C,WAAWkB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAAStD,SAAS,mBAAmB0C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMrD,aAAasC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIf,WACR,gBACA,CAAC,uBAAuB,EAAEgE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC7D,iBAAiBsC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACrB,YAAY0D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGxD,WAAWwD,GAAG,YAAY,EAAExD,WAAWwD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBAAgBc,MAAM;YAiEyCqD,iBAK7BA;QArEtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAEtE,WAAWoD,MAAM;QAErG,0FAA0F;QAC1F,sFAAsF;QACtF,uFAAuF;QACvF,eAAe;QACf,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,yFAAyF;YACzF,uFAAuF;YACvF,+DAA+D;YAC/D,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAP,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM+B,OAAOxB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIiE,OAAO;oBACxC,IAAIrB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAIkE,UAAU;oBACjD,QAAOlE,gBAAAA,IAAImE,IAAI,CAACvB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKuD,QAAQ,CAACd,WAAWe;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAM/D,aAAapB,gBAAgB4B,SAAS2D;YAC5C,MAAM5D,KAAKuD,QAAQ,CACjBjE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKuD,QAAQ,CACjB,4CACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAKuD,QAAQ,CACjB,6CACAM,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMtE,OAAOwC,WAAY,KAAK,MAAM+B,cAAcvE,IAAIwE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACzE,IAAIE,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE2E,WAAW7E,IAAI6E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM3F,QAAQiB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM1F,QAAQe,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASmD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG3C,IAAI,CAAC,OAAOwC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI3E,IAAI2E,IAAIH,KAAK,CAAC,KAAK/E,GAAG,CAAC,CAAC6E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAItG;IACpB,KAAK,MAAM,CAACuG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAEpE,GAAG,CAACuE,SAASC,KAAKF,QAAQrD,GAAG,CAACsD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAExE,GAAG,CAAC0E,MAAMD,QAAQrD,GAAG,CAACsD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAC/C,IAAMA,EAAEkC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQlE,IAAI,KAAK,IAAI,aAAa;WAAIkE;KAAQ,CAACrF,GAAG,CAAC+F,OAAO1D,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,eAAe2D,oBAAoBrG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMoG,OAAO,MAAMtG,KAAKQ,OAAO,CAAC;IAChC,MAAM+F,QAAQ,IAAInH,IAAI,AAAE,CAAA,MAAMkH,KAAK5F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ1B,UAAUwB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMiG,MAAMtF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAapB,gBAAgB4B,SAAS2D;IAC5C,MAAMjC,WAAqB,EAAE;IAC7B,MAAM6B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQrG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU8E,YAAY,EAAE,GAAG/H,UAAU8H;QAClD7E,SAASsC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC1E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBAAgBc,MAAM;QAC1B,IAAIwD,KAAK/B,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAACjE,oBAAoBkE;IAC/D;IACA,OAAO7B;AACT"}

package/dist/esm/watch.js

@@ -44,27 +44,38 @@ export async function runWatch(cfg, opts = {}) {
     };
     await touchHeartbeat();
     let debounceTimer = null;
+    // A reconcile that has already started owns the store, and on a bulk reparse a live worker
+    // pool as well. Shutdown waits on this rather than closing the connection underneath it.
+    let inFlight = null;
     const scheduleReconcile = ()=>{
         if (debounceTimer) clearTimeout(debounceTimer);
-        debounceTimer = setTimeout(async ()=>{
+        debounceTimer = setTimeout(()=>{
             debounceTimer = null;
-            try {
-                const { parsed, warnings } = await store.reconcile();
-                onEvent({
-                    type: 'reconciled',
-                    parsed,
-                    total: await docCount(store),
-                    warnings
-                });
-            } catch (err) {
-                onEvent({
-                    type: 'reconcile-error',
-                    message: err.message
-                });
-            }
+            inFlight = (async ()=>{
+                try {
+                    const { parsed, warnings } = await store.reconcile();
+                    onEvent({
+                        type: 'reconciled',
+                        parsed,
+                        total: await docCount(store),
+                        warnings
+                    });
+                } catch (err) {
+                    onEvent({
+                        type: 'reconcile-error',
+                        message: err.message
+                    });
+                } finally{
+                    inFlight = null;
+                }
+            })();
         }, debounceMs);
     };
-    // Ignore our own state dir, or the heartbeat write would retrigger itself forever.
+    // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event
+    // whose filename the platform could not resolve (null, which fs.watch does deliver under
+    // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs
+    // less than missing a real edit. That is why the guard cannot promise zero reconciles, only
+    // that an identified state-dir write is never one of them.
     const watcher = fsWatch(baseDir, {
         recursive: true
     }, (_event, filename)=>{
@@ -86,6 +97,7 @@ export async function runWatch(cfg, opts = {}) {
             clearInterval(heartbeatTimer);
             if (debounceTimer) clearTimeout(debounceTimer);
             watcher.close();
+            await inFlight;
             await setMeta(store, 'watch_heartbeat', null);
             await setMeta(store, 'watch_pid', null);
             await store.close();

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(async () => {\n      debounceTimer = null;\n      try {\n        const { parsed, warnings } = await store.reconcile();\n        onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAElG,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUO;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMpB,QAAQa,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMhC,SAASc;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,MAAMC,oBAAoB;QACxB,IAAID,eAAeE,aAAaF;QAChCA,gBAAgBG,WAAW;YACzBH,gBAAgB;YAChB,IAAI;gBACF,MAAM,EAAErB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM6B,SAAS;gBAClDhC,QAAQ;oBAAEmB,MAAM;oBAAcZ;oBAAQc,OAAO,MAAMhC,SAASc;oBAAQE;gBAAS;YAC/E,EAAE,OAAO4B,KAAK;gBACZjC,QAAQ;oBAAEmB,MAAM;oBAAmBe,SAAS,AAACD,IAAcC,OAAO;gBAAC;YACrE;QACF,GAAGjC;IACL;IAEA,mFAAmF;IACnF,MAAMkC,UAAUlD,QAAQwB,SAAS;QAAE2B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACrD,YAAY;QACpE2C;IACF;IACA,MAAMW,iBAAiBC,YACrBrD,YAAYmC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIwC,QAAc,CAACC;YAmBpB5C,cACCA;QAnBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM6C,WAAW;gBAKf7C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQmB,GAAG,CAAC,UAAUD;YACtBlB,QAAQmB,GAAG,CAAC,WAAWD;aACvB7C,eAAAA,KAAK+C,MAAM,cAAX/C,mCAAAA,aAAagD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIZ,eAAeE,aAAaF;YAChCO,QAAQnB,KAAK;YACb,MAAMvB,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB2B;QACF;QACAjB,QAAQuB,IAAI,CAAC,UAAUL;QACvBlB,QAAQuB,IAAI,CAAC,WAAWL;QACxB,KAAI7C,eAAAA,KAAK+C,MAAM,cAAX/C,mCAAAA,aAAamD,OAAO,EAAEN;cACrB7C,gBAAAA,KAAK+C,MAAM,cAAX/C,oCAAAA,cAAaoD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event\n  // whose filename the platform could not resolve (null, which fs.watch does deliver under\n  // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs\n  // less than missing a real edit. That is why the guard cannot promise zero reconciles, only\n  // that an identified state-dir write is never one of them.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","inFlight","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAElG,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUO;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMpB,QAAQa,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMhC,SAASc;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAIC,WAAiC;IACrC,MAAMC,oBAAoB;QACxB,IAAIF,eAAeG,aAAaH;QAChCA,gBAAgBI,WAAW;YACzBJ,gBAAgB;YAChBC,WAAW,AAAC,CAAA;gBACV,IAAI;oBACF,MAAM,EAAEtB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM8B,SAAS;oBAClDjC,QAAQ;wBAAEmB,MAAM;wBAAcZ;wBAAQc,OAAO,MAAMhC,SAASc;wBAAQE;oBAAS;gBAC/E,EAAE,OAAO6B,KAAK;oBACZlC,QAAQ;wBAAEmB,MAAM;wBAAmBgB,SAAS,AAACD,IAAcC,OAAO;oBAAC;gBACrE,SAAU;oBACRN,WAAW;gBACb;YACF,CAAA;QACF,GAAG5B;IACL;IAEA,4FAA4F;IAC5F,yFAAyF;IACzF,4FAA4F;IAC5F,4FAA4F;IAC5F,2DAA2D;IAC3D,MAAMmC,UAAUnD,QAAQwB,SAAS;QAAE4B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACtD,YAAY;QACpE4C;IACF;IACA,MAAMW,iBAAiBC,YACrBtD,YAAYmC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIyC,QAAc,CAACC;YAoBpB7C,cACCA;QApBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM8C,WAAW;gBAKf9C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQoB,GAAG,CAAC,UAAUD;YACtBnB,QAAQoB,GAAG,CAAC,WAAWD;aACvB9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaiD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIb,eAAeG,aAAaH;YAChCQ,QAAQpB,KAAK;YACb,MAAMa;YACN,MAAMpC,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB4B;QACF;QACAlB,QAAQwB,IAAI,CAAC,UAAUL;QACvBnB,QAAQwB,IAAI,CAAC,WAAWL;QACxB,KAAI9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaoD,OAAO,EAAEN;cACrB9C,gBAAAA,KAAKgD,MAAM,cAAXhD,oCAAAA,cAAaqD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}

package/dist/esm/workers/parse.d.ts

@@ -0,0 +1,18 @@
+import type { Config, FeatureName } from '../config/index.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from '../scan/list.js';
+import type { WorkerErrorPayload } from '../scan/worker-error.js';
+export interface ParseWorkerData {
+    cfg: Config;
+    featureNames: FeatureName[];
+}
+export type ParseTask = FileStat;
+export type ParseTaskResult = {
+    ok: true;
+    doc: ParsedDoc;
+    warnings: string[];
+} | {
+    ok: false;
+    error: WorkerErrorPayload;
+};
+export default function parseTask(file: ParseTask): ParseTaskResult;

package/dist/esm/workers/parse.js

@@ -0,0 +1,25 @@
+import Tinypool from 'tinypool';
+import { FEATURES } from '../features/index.js';
+import { parseFile } from '../scan/index.js';
+import { featuresForFile } from '../scan/reparse.js';
+import { serializeError } from '../scan/worker-error.js';
+// Read once per worker, not per task.
+const { cfg, featureNames } = Tinypool.workerData;
+// Filtering the registry (rather than mapping the names) keeps registry order, which is the
+// order `extracted` keys land in on the serial path.
+const selected = FEATURES.filter((feature)=>featureNames.includes(feature.name));
+export default function parseTask(file) {
+    try {
+        const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);
+        return {
+            ok: true,
+            doc,
+            warnings
+        };
+    } catch (err) {
+        return {
+            ok: false,
+            error: serializeError(err)
+        };
+    }
+}

package/dist/esm/workers/parse.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task.\n// A Feature carries closures and cannot cross the thread boundary; its name can, and the\n// registry on this side resolves it back, which keeps the caller's selection intact.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[] } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    return { ok: true, doc, warnings };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["Tinypool","FEATURES","parseFile","featuresForFile","serializeError","cfg","featureNames","workerData","selected","filter","feature","includes","name","parseTask","file","doc","warnings","ok","err","error"],"mappings":"AAAA,OAAOA,cAAc,WAAW;AAEhC,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,eAAe,QAAQ,qBAAqB;AAErD,SAASC,cAAc,QAAQ,0BAA0B;AAgBzD,sCAAsC;AACtC,MAAM,EAAEC,GAAG,EAAEC,YAAY,EAAE,GAAGN,SAASO,UAAU;AACjD,4FAA4F;AAC5F,qDAAqD;AACrD,MAAMC,WAAWP,SAASQ,MAAM,CAAC,CAACC,UAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;AAEhF,eAAe,SAASC,UAAUC,IAAe;IAC/C,IAAI;QACF,MAAM,EAAEC,GAAG,EAAEC,QAAQ,EAAE,GAAGd,UAAUY,MAAMX,gBAAgBK,UAAUH,KAAKS,OAAOT;QAChF,OAAO;YAAEY,IAAI;YAAMF;YAAKC;QAAS;IACnC,EAAE,OAAOE,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOf,eAAec;QAAK;IACjD;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.18.1",
+  "version": "0.18.2",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -92,6 +92,7 @@
     "micromark-extension-gfm-strikethrough": "^2.0.0",
     "micromark-extension-gfm-table": "^2.0.0",
     "micromark-extension-gfm-task-list-item": "^2.0.0",
+    "tinypool": "^2.1.2",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
@@ -100,6 +101,7 @@
     "@types/mocha": "*",
     "@types/node": "*",
     "cr": "^0.1.0",
+    "fs-remove-compat": "^1.0.4",
     "node-version-use": "*",
     "ts-dev-stack": "*",
     "tsds-config": "*"