sensemaking 0.19.2 → 0.21.0

package/dist/cjs/scan/reparse.js

@@ -16,11 +16,9 @@ _export(exports, {
         return reparseFiles;
     }
 });
-var _nodemodule = require("node:module");
 var _nodeos = require("node:os");
-var _nodepath = require("node:path");
 var _indexts = require("./index.js");
-var _workererrorts = require("./worker-error.js");
+var _poolts = require("./pool.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -173,34 +171,10 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
-// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.
-var _require = typeof require === 'undefined' ? (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString()) : require;
 // Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files
 // (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.
 var DEFAULT_THRESHOLD = 200;
 var DEFAULT_MAX_WORKERS = 8;
-// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited
-// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.
-var workerFile;
-function resolveWorkerFile() {
-    if (workerFile) return workerFile;
-    var load = (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString());
-    for(var _i = 0, _iter = [
-        '..',
-        '../..',
-        '../../..'
-    ]; _i < _iter.length; _i++){
-        var rel = _iter[_i];
-        try {
-            if (load("".concat(rel, "/package.json")).name === 'sensemaking') {
-                workerFile = (0, _nodepath.join)((0, _nodepath.dirname)(load.resolve("".concat(rel, "/package.json"))), 'dist', 'cjs', 'workers', 'parse.js');
-                return workerFile;
-            }
-        } catch (unused) {}
-    }
-    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
-}
 function featuresForFile(features, cfg, file) {
     return features.filter(function(feature) {
         return !feature.enabledForFile || feature.enabledForFile(cfg, file);
@@ -233,30 +207,19 @@ function reparseSerial(files, features, cfg, onParsed) {
     }
     return results;
 }
-// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
-// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.
-function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
+// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with
+// none given, opens an ephemeral ParsePool for this call alone and always closes it.
+function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
     return _async_to_generator(function() {
-        var Tinypool, workerData, pool, done, results, err;
+        var ephemeral, results, err;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    Tinypool = _require('tinypool').Tinypool;
-                    // cfg and the feature selection are constant for the dispatch, so they cross once per worker
-                    // as workerData. Features carry closures and cannot cross at all; only their names do.
-                    workerData = {
-                        cfg: cfg,
-                        featureNames: features.map(function(feature) {
-                            return feature.name;
-                        })
-                    };
-                    pool = new Tinypool({
-                        filename: resolveWorkerFile(),
-                        minThreads: maxWorkers,
-                        maxThreads: maxWorkers,
-                        workerData: workerData
-                    });
-                    done = 0;
+                    if (pool) return [
+                        2,
+                        pool.run(files, features, cfg, onParsed, maxWorkers)
+                    ];
+                    ephemeral = new _poolts.ParsePool();
                     _state.label = 1;
                 case 1:
                     _state.trys.push([
@@ -267,37 +230,13 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
                     ]);
                     return [
                         4,
-                        Promise.all(files.map(function(file) {
-                            return _async_to_generator(function() {
-                                var result;
-                                return _ts_generator(this, function(_state) {
-                                    switch(_state.label){
-                                        case 0:
-                                            return [
-                                                4,
-                                                pool.run(file)
-                                            ];
-                                        case 1:
-                                            result = _state.sent();
-                                            if (!result.ok) throw (0, _workererrorts.reviveError)(result.error);
-                                            onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
-                                            return [
-                                                2,
-                                                {
-                                                    doc: result.doc,
-                                                    warnings: result.warnings
-                                                }
-                                            ];
-                                    }
-                                });
-                            })();
-                        }))
+                        ephemeral.run(files, features, cfg, onParsed, maxWorkers)
                     ];
                 case 2:
                     results = _state.sent();
                     return [
                         4,
-                        pool.destroy()
+                        ephemeral.close()
                     ];
                 case 3:
                     _state.sent();
@@ -307,11 +246,11 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
                     ];
                 case 4:
                     err = _state.sent();
-                    // Tear the pool down before rethrowing, but a destroy failure must not mask the parse
+                    // Tear the pool down before rethrowing, but a close failure must not mask the parse
                     // failure that caused it -- not a `finally`, for that reason.
                     return [
                         4,
-                        pool.destroy().catch(function() {})
+                        ephemeral.close().catch(function() {})
                     ];
                 case 5:
                     _state.sent();
@@ -340,7 +279,7 @@ function reparseFiles(_0, _1, _2, _3, _4) {
                     ];
                     return [
                         4,
-                        reparsePooled(files, features, cfg, onParsed, maxWorkers)
+                        reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool)
                     ];
                 case 1:
                     _tmp = _state.sent();

package/dist/cjs/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\n// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n  // as workerData. Features carry closures and cannot cross at all; only their names do.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a destroy failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["featuresForFile","reparseFiles","_require","require","createRequire","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","Tinypool","workerData","pool","err","featureNames","map","filename","minThreads","maxThreads","Promise","all","result","run","ok","reviveError","error","doc","warnings","destroy","catch","knownColumns","options","threshold","docs","newColumns","seen","fileWarnings","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QA6DgBA;eAAAA;;QA+CMC;eAAAA;;;0BA5GQ;sBACO;wBACP;uBAQJ;6BAEE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,qIAAqI;AACrI,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAmBnF,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAME,oBAAoB;AAC1B,IAAMC,sBAAsB;AAE5B,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOL,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCM,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAMO,SAASf,gBAAgBgB,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAe3B,gBAAgBgB,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB;;YAC1IC,UAGFC,YACAC,MACFR,MAIID,SAUCU;;;;oBAnBDH,WAAa9B,SAAS,YAAtB8B;oBACR,6FAA6F;oBAC7F,uFAAuF;oBACjFC,aAA8B;wBAAEhB,KAAAA;wBAAKmB,cAAcpB,SAASqB,GAAG,CAAC,SAACjB;mCAAYA,QAAQT,IAAI;;oBAAE;oBAC3FuB,OAAO,IAAIF,SAAS;wBAAEM,UAAU9B;wBAAqB+B,YAAYR;wBAAYS,YAAYT;wBAAYE,YAAAA;oBAAW;oBAClHP,OAAO;;;;;;;;;oBAIO;;wBAAMe,QAAQC,GAAG,CAC/BnB,MAAMc,GAAG,CAAC,SAAOnB;;oCACTyB;;;;4CAAU;;gDAAMT,KAAKU,GAAG,CAAC1B;;;4CAAzByB,SAAU;4CAChB,IAAI,CAACA,OAAOE,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACH,OAAOI,KAAK;4CAC9CvB,qBAAAA,+BAAAA,SAAW,EAAEE;4CACb;;gDAAO;oDAAEsB,KAAKL,OAAOK,GAAG;oDAAEC,UAAUN,OAAOM,QAAQ;gDAAC;;;;4BACtD;;;;oBANIxB,UAAU;oBAQhB;;wBAAMS,KAAKgB,OAAO;;;oBAAlB;oBACA;;wBAAOzB;;;oBACAU;oBACP,sFAAsF;oBACtF,8DAA8D;oBAC9D;;wBAAMD,KAAKgB,OAAO,GAAGC,KAAK,CAAC,YAAO;;;oBAAlC;oBACA,MAAMhB;;;;;;;IAEV;;AAIO,SAAelC;wCAAasB,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEmC,YAAiC,EAAE5B,QAAiC;YAAE6B,SAC1IA,oBACCA,qBADbC,WACAvB,YACAN,eAEA8B,MACAN,UACAO,YACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,oBAAQT,KAAeU,cAC1BT,WACK,4BAAA,oBAAA,iBAAA,YAAA,QAAMU;;;;;oBAX+IN,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqBhD;oBACjC0B,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBO,KAAKC,GAAG,CAACvD,qBAAqBwD,IAAAA,4BAAoB;yBAC3EvC,CAAAA,MAAMwC,MAAM,IAAIT,SAAQ,GAAxB/B;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO;;;2BAApD;;;;;;2BAAkET,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA5IC;oBAEA8B;oBACAN;oBACAO;oBACAC,OAAO,IAAIO,IAAIZ;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAyC3B,8BAAzC,6BAAA,QAAA,yBAAA,iCAAkD;0CAAlD,aAAQuB,kBAAAA,KAAeU,2BAAVT;;4BAChBA,CAAAA,YAAAA,UAASrB,IAAI,OAAbqB,WAAc,qBAAGS;4BACZ,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAAClB,IAAImB,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACF,KAAKW,GAAG,CAACT,MAAM;wCAClBF,KAAKY,GAAG,CAACV;wCACTH,WAAW5B,IAAI,CAAC+B;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLJ,KAAK3B,IAAI,CAACoB;wBACZ;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWL;;wBAAO;4BAAEO,MAAAA;4BAAMN,UAAAA;4BAAUO,YAAAA;wBAAW;;;;IACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["featuresForFile","reparseFiles","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","err","run","ParsePool","close","catch","knownColumns","options","threshold","docs","warnings","newColumns","seen","doc","fileWarnings","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAoCgBA;eAAAA;;QAkCMC;eAAAA;;;sBAtEe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsB1B,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAMC,oBAAoB;AAC1B,IAAMC,sBAAsB;AAIrB,SAASH,gBAAgBI,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAef,gBAAgBI,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WAEER,SAGCS;;;;oBANT,IAAIF,MAAM;;wBAAOA,KAAKG,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIG,iBAAS;;;;;;;;;oBAEb;;wBAAMH,UAAUE,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;oBAChB;;wBAAMQ,UAAUI,KAAK;;;oBAArB;oBACA;;wBAAOZ;;;oBACAS;oBACP,oFAAoF;oBACpF,8DAA8D;oBAC9D;;wBAAMD,UAAUI,KAAK,GAAGC,KAAK,CAAC,YAAO;;;oBAArC;oBACA,MAAMJ;;;;;;;IAEV;;AAIO,SAAerB;wCAAaU,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEsB,YAAiC,EAAEf,QAAiC;YAAEgB,SAC1IA,oBACCA,qBADbC,WACAV,YACAN,eAEAiB,MACAC,UACAC,YACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAC1BJ,WACK,4BAAA,oBAAA,iBAAA,YAAA,QAAMK;;;;;oBAX+IR,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;oBACjCiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBS,KAAKC,GAAG,CAACnC,qBAAqBoC,IAAAA,4BAAoB;yBAC3E5B,CAAAA,MAAM6B,MAAM,IAAIX,SAAQ,GAAxBlB;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO,YAAYS,QAAQR,IAAI;;;2BAA5E;;;;;;2BAAgFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA1JC;oBAEAiB;oBACAC;oBACAC;oBACAC,OAAO,IAAIQ,IAAId;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAyCd,8BAAzC,6BAAA,QAAA,yBAAA,iCAAkD;0CAAlD,aAAQqB,kBAAAA,KAAeC,2BAAVJ;;4BAChBA,CAAAA,YAAAA,UAASf,IAAI,OAAbe,WAAc,qBAAGI;4BACZ,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAACT,IAAIU,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACH,KAAKY,GAAG,CAACT,MAAM;wCAClBH,KAAKa,GAAG,CAACV;wCACTJ,WAAWhB,IAAI,CAACoB;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLN,KAAKd,IAAI,CAACkB;wBACZ;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWL;;wBAAO;4BAAEJ,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;wBAAW;;;;IACtC"}

package/dist/cjs/store/builder.d.cts

@@ -0,0 +1,19 @@
+import type { Config } from '../config/index.js';
+import type { EmbedChangeKind } from './embed-scope.js';
+import type { FeatureToggle } from './feature-scope.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export interface Builder {
+    build(forcedPaths?: ReadonlySet<string>): Promise<{
+        parsed: number;
+        warnings: string[];
+    }>;
+    invalidate(kind: EmbedChangeKind): Promise<{
+        parsed: number;
+    }>;
+    invalidateFeatures(toggles: FeatureToggle[]): Promise<{
+        parsed: number;
+    }>;
+    close(): Promise<void>;
+    readonly poolsCreated: number;
+}
+export declare function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder;

package/dist/cjs/store/builder.d.ts

@@ -0,0 +1,19 @@
+import type { Config } from '../config/index.js';
+import type { EmbedChangeKind } from './embed-scope.js';
+import type { FeatureToggle } from './feature-scope.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export interface Builder {
+    build(forcedPaths?: ReadonlySet<string>): Promise<{
+        parsed: number;
+        warnings: string[];
+    }>;
+    invalidate(kind: EmbedChangeKind): Promise<{
+        parsed: number;
+    }>;
+    invalidateFeatures(toggles: FeatureToggle[]): Promise<{
+        parsed: number;
+    }>;
+    close(): Promise<void>;
+    readonly poolsCreated: number;
+}
+export declare function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder;