sensemaking 0.25.1 → 0.27.0

package/dist/cjs/scan/reparse.js

@@ -9,6 +9,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get PARSE_BYTES_PER_WORKER () {
+        return PARSE_BYTES_PER_WORKER;
+    },
     get featuresForFile () {
         return featuresForFile;
     },
@@ -171,9 +174,7 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files
-// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.
-var DEFAULT_THRESHOLD = 200;
+var PARSE_BYTES_PER_WORKER = 512 * 1024;
 var DEFAULT_MAX_WORKERS = 8;
 function featuresForFile(features, cfg, file) {
     return features.filter(function(feature) {
@@ -211,7 +212,7 @@ function reparseSerial(files, features, cfg, onParsed) {
 // none given, opens an ephemeral ParsePool for this call alone and always closes it.
 function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
     return _async_to_generator(function() {
-        var ephemeral, results, err;
+        var ephemeral, results, dispatchError, dispatchFailed, err, closeError;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -220,13 +221,14 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
                         pool.run(files, features, cfg, onParsed, maxWorkers)
                     ];
                     ephemeral = new _poolts.ParsePool();
+                    dispatchFailed = false;
                     _state.label = 1;
                 case 1:
                     _state.trys.push([
                         1,
-                        4,
+                        3,
                         ,
-                        6
+                        4
                     ]);
                     return [
                         4,
@@ -235,51 +237,103 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
                 case 2:
                     results = _state.sent();
                     return [
-                        4,
-                        ephemeral.close()
+                        3,
+                        4
                     ];
                 case 3:
-                    _state.sent();
+                    err = _state.sent();
+                    dispatchFailed = true;
+                    dispatchError = err;
                     return [
-                        2,
-                        results
+                        3,
+                        4
                     ];
                 case 4:
-                    err = _state.sent();
-                    // Tear the pool down before rethrowing, but a close failure must not mask the parse
-                    // failure that caused it -- not a `finally`, for that reason.
+                    _state.trys.push([
+                        4,
+                        6,
+                        ,
+                        7
+                    ]);
                     return [
                         4,
-                        ephemeral.close().catch(function() {})
+                        ephemeral.close()
                     ];
                 case 5:
                     _state.sent();
-                    throw err;
+                    return [
+                        3,
+                        7
+                    ];
                 case 6:
+                    closeError = _state.sent();
+                    if (dispatchFailed) throw new AggregateError([
+                        dispatchError,
+                        closeError
+                    ], 'parse dispatch and pool close failed');
+                    throw closeError;
+                case 7:
+                    if (dispatchFailed) throw dispatchError;
+                    if (results === undefined) throw new Error('parse dispatch finished without results');
                     return [
-                        2
+                        2,
+                        results
                     ];
             }
         });
     })();
 }
+function parseWorkerCount(files, bytesPerWorker, maxWorkers) {
+    if (files.length === 0) return 0;
+    var available = Math.max(1, Math.min(maxWorkers, files.length));
+    if (bytesPerWorker === 0) return available;
+    var totalBytes = 0;
+    var largestBytes = 0;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = files[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var file = _step.value;
+            totalBytes += file.size;
+            largestBytes = Math.max(largestBytes, file.size);
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    var aggregateWorkers = Math.floor(totalBytes / bytesPerWorker);
+    // The largest file is indivisible, so extra workers need full shares among the remainder.
+    var companionWorkers = 1 + Math.floor((totalBytes - largestBytes) / bytesPerWorker);
+    return Math.min(available, aggregateWorkers, companionWorkers);
+}
 function reparseFiles(_0, _1, _2, _3, _4) {
     return _async_to_generator(function(files, features, cfg, knownColumns, onParsed) {
-        var options, _options_threshold, _options_maxWorkers, threshold, maxWorkers, results, _tmp, docs, warnings, newColumns, seen, workerParseMs, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, doc, fileWarnings, parseMs, _warnings, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, key;
+        var options, _options_threshold, _options_maxWorkers, threshold, maxWorkers, workerCount, pooled, results, _tmp, docs, warnings, newColumns, seen, workerParseMs, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, doc, fileWarnings, parseMs, _warnings, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, key;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
                     options = _arguments.length > 5 && _arguments[5] !== void 0 ? _arguments[5] : {};
-                    threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold : DEFAULT_THRESHOLD;
+                    threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold : PARSE_BYTES_PER_WORKER;
                     maxWorkers = (_options_maxWorkers = options.maxWorkers) !== null && _options_maxWorkers !== void 0 ? _options_maxWorkers : Math.min(DEFAULT_MAX_WORKERS, (0, _nodeos.availableParallelism)());
-                    if (!(files.length >= threshold)) return [
+                    workerCount = parseWorkerCount(files, threshold, maxWorkers);
+                    pooled = files.length > 0 && (threshold === 0 || workerCount >= 2);
+                    if (!pooled) return [
                         3,
                         2
                     ];
                     return [
                         4,
-                        reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool)
+                        reparsePooled(files, features, cfg, onParsed, workerCount, options.pool)
                     ];
                 case 1:
                     _tmp = _state.sent();

package/dist/cjs/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n  // Sum of worker-side parseFile time (workers/parse.ts) across files; 0 on the serial path.\n  workerParseMs: number;\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  let workerParseMs = 0;\n  for (const { doc, warnings: fileWarnings, parseMs } of results) {\n    warnings.push(...fileWarnings);\n    workerParseMs += parseMs ?? 0;\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns, workerParseMs };\n}\n"],"names":["featuresForFile","reparseFiles","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","err","run","ParsePool","close","catch","knownColumns","options","threshold","docs","warnings","newColumns","seen","workerParseMs","doc","fileWarnings","parseMs","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAsCgBA;eAAAA;;QAkCMC;eAAAA;;;sBAxEe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwB1B,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAMC,oBAAoB;AAC1B,IAAMC,sBAAsB;AAIrB,SAASH,gBAAgBI,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAef,gBAAgBI,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WAEER,SAGCS;;;;oBANT,IAAIF,MAAM;;wBAAOA,KAAKG,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIG,iBAAS;;;;;;;;;oBAEb;;wBAAMH,UAAUE,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;oBAChB;;wBAAMQ,UAAUI,KAAK;;;oBAArB;oBACA;;wBAAOZ;;;oBACAS;oBACP,oFAAoF;oBACpF,8DAA8D;oBAC9D;;wBAAMD,UAAUI,KAAK,GAAGC,KAAK,CAAC,YAAO;;;oBAArC;oBACA,MAAMJ;;;;;;;IAEV;;AAIO,SAAerB;wCAAaU,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEsB,YAAiC,EAAEf,QAAiC;YAAEgB,SAC1IA,oBACCA,qBADbC,WACAV,YACAN,eAEAiB,MACAC,UACAC,YACAC,MACFC,eACC,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAAcC,SACxCN,WAEK,4BAAA,oBAAA,iBAAA,YAAA,QAAMO;;;;;oBAb+IV,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;oBACjCiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBW,KAAKC,GAAG,CAACrC,qBAAqBsC,IAAAA,4BAAoB;yBAC3E9B,CAAAA,MAAM+B,MAAM,IAAIb,SAAQ,GAAxBlB;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO,YAAYS,QAAQR,IAAI;;;2BAA5E;;;;;;2BAAgFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA1JC;oBAEAiB;oBACAC;oBACAC;oBACAC,OAAO,IAAIU,IAAIhB;oBACjBO,gBAAgB;oBACf,kCAAA,2BAAA;;wBAAL,IAAK,YAAkDrB,8BAAlD,6BAAA,QAAA,yBAAA,iCAA2D;0CAA3D,aAAQsB,kBAAAA,KAAeC,2BAAVL,UAAwBM,sBAAAA;;4BACxCN,CAAAA,YAAAA,UAASf,IAAI,OAAbe,WAAc,qBAAGK;4BACjBF,iBAAiBG,oBAAAA,qBAAAA,UAAW;4BACvB,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAACV,IAAIW,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACL,KAAKc,GAAG,CAACT,MAAM;wCAClBL,KAAKe,GAAG,CAACV;wCACTN,WAAWhB,IAAI,CAACsB;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLR,KAAKd,IAAI,CAACmB;wBACZ;;wBAVK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL;;wBAAO;4BAAEL,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;4BAAYE,eAAAA;wBAAc;;;;IACrD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n  // Sum of worker-side parseFile time (workers/parse.ts) across files; 0 on the serial path.\n  workerParseMs: number;\n}\n\nexport interface ReparseOptions {\n  // Overrides PARSE_BYTES_PER_WORKER below. Zero forces worker dispatch in tests.\n  // Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// The 2026-09-26 comparison kept ~326 KiB updates serial and 5.86 MiB work pooled.\n// Companion byte shares prevent one dominant file from selecting workers for a tiny remainder.\nexport const PARSE_BYTES_PER_WORKER = 512 * 1024;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  let results: FileResult[] | undefined;\n  let dispatchError: unknown;\n  let dispatchFailed = false;\n  try {\n    results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n  } catch (err) {\n    dispatchFailed = true;\n    dispatchError = err;\n  }\n  try {\n    await ephemeral.close();\n  } catch (closeError) {\n    if (dispatchFailed) throw new AggregateError([dispatchError, closeError], 'parse dispatch and pool close failed');\n    throw closeError;\n  }\n  if (dispatchFailed) throw dispatchError;\n  if (results === undefined) throw new Error('parse dispatch finished without results');\n  return results;\n}\n\nfunction parseWorkerCount(files: FileStat[], bytesPerWorker: number, maxWorkers: number): number {\n  if (files.length === 0) return 0;\n  const available = Math.max(1, Math.min(maxWorkers, files.length));\n  if (bytesPerWorker === 0) return available;\n  let totalBytes = 0;\n  let largestBytes = 0;\n  for (const file of files) {\n    totalBytes += file.size;\n    largestBytes = Math.max(largestBytes, file.size);\n  }\n  const aggregateWorkers = Math.floor(totalBytes / bytesPerWorker);\n  // The largest file is indivisible, so extra workers need full shares among the remainder.\n  const companionWorkers = 1 + Math.floor((totalBytes - largestBytes) / bytesPerWorker);\n  return Math.min(available, aggregateWorkers, companionWorkers);\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? PARSE_BYTES_PER_WORKER;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const workerCount = parseWorkerCount(files, threshold, maxWorkers);\n  const pooled = files.length > 0 && (threshold === 0 || workerCount >= 2);\n  const results = pooled ? await reparsePooled(files, features, cfg, onParsed, workerCount, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  let workerParseMs = 0;\n  for (const { doc, warnings: fileWarnings, parseMs } of results) {\n    warnings.push(...fileWarnings);\n    workerParseMs += parseMs ?? 0;\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns, workerParseMs };\n}\n"],"names":["PARSE_BYTES_PER_WORKER","featuresForFile","reparseFiles","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","dispatchError","dispatchFailed","err","closeError","run","ParsePool","close","AggregateError","undefined","Error","parseWorkerCount","bytesPerWorker","length","available","Math","max","min","totalBytes","largestBytes","size","aggregateWorkers","floor","companionWorkers","knownColumns","options","threshold","workerCount","pooled","docs","warnings","newColumns","seen","workerParseMs","doc","fileWarnings","parseMs","key","availableParallelism","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAiCaA;eAAAA;;QAKGC;eAAAA;;QA0DMC;eAAAA;;;sBAhGe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0BnB,IAAMF,yBAAyB,MAAM;AAC5C,IAAMG,sBAAsB;AAIrB,SAASF,gBAAgBG,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAed,gBAAgBG,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WACFR,SACAS,eACAC,gBAGKC,KAMAC;;;;oBAbT,IAAIL,MAAM;;wBAAOA,KAAKM,GAAG,CAACf,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIM,iBAAS;oBAG3BJ,iBAAiB;;;;;;;;;oBAET;;wBAAMF,UAAUK,GAAG,CAACf,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;;;;;;oBACHW;oBACPD,iBAAiB;oBACjBD,gBAAgBE;;;;;;;;;;;;oBAGhB;;wBAAMH,UAAUO,KAAK;;;oBAArB;;;;;;oBACOH;oBACP,IAAIF,gBAAgB,MAAM,IAAIM;wBAAgBP;wBAAeG;uBAAa;oBAC1E,MAAMA;;oBAER,IAAIF,gBAAgB,MAAMD;oBAC1B,IAAIT,YAAYiB,WAAW,MAAM,IAAIC,MAAM;oBAC3C;;wBAAOlB;;;;IACT;;AAEA,SAASmB,iBAAiBrB,KAAiB,EAAEsB,cAAsB,EAAEd,UAAkB;IACrF,IAAIR,MAAMuB,MAAM,KAAK,GAAG,OAAO;IAC/B,IAAMC,YAAYC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,GAAG,CAACnB,YAAYR,MAAMuB,MAAM;IAC/D,IAAID,mBAAmB,GAAG,OAAOE;IACjC,IAAII,aAAa;IACjB,IAAIC,eAAe;QACd,kCAAA,2BAAA;;QAAL,QAAK,YAAc7B,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACHiC,cAAcjC,KAAKmC,IAAI;YACvBD,eAAeJ,KAAKC,GAAG,CAACG,cAAclC,KAAKmC,IAAI;QACjD;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,IAAMC,mBAAmBN,KAAKO,KAAK,CAACJ,aAAaN;IACjD,0FAA0F;IAC1F,IAAMW,mBAAmB,IAAIR,KAAKO,KAAK,CAAC,AAACJ,CAAAA,aAAaC,YAAW,IAAKP;IACtE,OAAOG,KAAKE,GAAG,CAACH,WAAWO,kBAAkBE;AAC/C;AAIO,SAAe1C;wCAAaS,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEwC,YAAiC,EAAEjC,QAAiC;YAAEkC,SAC1IA,oBACCA,qBADbC,WACA5B,YACA6B,aACAC,QACApC,eAEAqC,MACAC,UACAC,YACAC,MACFC,eACC,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAAcC,SACxCN,WAEK,4BAAA,oBAAA,iBAAA,YAAA,QAAMO;;;;;oBAf+IZ,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB9C;oBACjCmB,cAAa2B,sBAAAA,QAAQ3B,UAAU,cAAlB2B,iCAAAA,sBAAsBV,KAAKE,GAAG,CAACnC,qBAAqBwD,IAAAA,4BAAoB;oBACrFX,cAAchB,iBAAiBrB,OAAOoC,WAAW5B;oBACjD8B,SAAStC,MAAMuB,MAAM,GAAG,KAAMa,CAAAA,cAAc,KAAKC,eAAe,CAAA;yBACtDC,QAAAA;;;;oBAAS;;wBAAM/B,cAAcP,OAAOP,UAAUC,KAAKO,UAAUoC,aAAaF,QAAQ1B,IAAI;;;2BAA7E;;;;;;2BAAiFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAAxIC;oBAEAqC;oBACAC;oBACAC;oBACAC,OAAO,IAAIO,IAAIf;oBACjBS,gBAAgB;oBACf,kCAAA,2BAAA;;wBAAL,IAAK,YAAkDzC,8BAAlD,6BAAA,QAAA,yBAAA,iCAA2D;0CAA3D,aAAQ0C,kBAAAA,KAAeC,2BAAVL,UAAwBM,sBAAAA;;4BACxCN,CAAAA,YAAAA,UAASnC,IAAI,OAAbmC,WAAc,qBAAGK;4BACjBF,iBAAiBG,oBAAAA,qBAAAA,UAAW;4BACvB,mCAAA,4BAAA;;gCAAL,IAAK,aAAaI,OAAOC,IAAI,CAACP,IAAIQ,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BL,MAAN;oCACH,IAAI,CAACL,KAAKW,GAAG,CAACN,MAAM;wCAClBL,KAAKY,GAAG,CAACP;wCACTN,WAAWpC,IAAI,CAAC0C;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLR,KAAKlC,IAAI,CAACuC;wBACZ;;wBAVK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL;;wBAAO;4BAAEL,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;4BAAYE,eAAAA;wBAAc;;;;IACrD"}

package/dist/cjs/store/builder.d.cts

@@ -1,27 +1,17 @@
-import { type Config } from '../config/index.js';
-import type { EmbedChangeKind } from './embed-scope.js';
-import type { FeatureToggle } from './feature-scope.js';
-import { type ReconcilePaths } from './reconcile.js';
+import { type ResolvedConfig } from '../config/index.js';
 import type { Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
+interface BuildResult {
+    parsed: number;
+    warnings: string[];
+    stages: Stages;
+    messages: string[];
+}
 export interface Builder {
-    build(forcedPaths?: ReadonlySet<string>): Promise<{
-        parsed: number;
-        warnings: string[];
-        stages: Stages;
-    }>;
-    recover(): Promise<{
-        parsed: number;
-        warnings: string[];
-        stages: Stages;
-    }>;
-    invalidate(kind: EmbedChangeKind): Promise<{
-        parsed: number;
-    }>;
-    invalidateFeatures(toggles: FeatureToggle[]): Promise<{
-        parsed: number;
-    }>;
+    build(): Promise<BuildResult>;
     close(): Promise<void>;
     readonly poolsCreated: number;
 }
-export declare function createBuilder(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect): Builder;
+type EnsureFeatureSchema = (cfg: ResolvedConfig) => Promise<void>;
+export declare function createBuilder(conn: Connection, cfg: ResolvedConfig, dialect: ReconcileDialect, ensureFeatureSchema: EnsureFeatureSchema): Builder;
+export {};

package/dist/cjs/store/builder.d.ts

@@ -1,27 +1,17 @@
-import { type Config } from '../config/index.js';
-import type { EmbedChangeKind } from './embed-scope.js';
-import type { FeatureToggle } from './feature-scope.js';
-import { type ReconcilePaths } from './reconcile.js';
+import { type ResolvedConfig } from '../config/index.js';
 import type { Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
+interface BuildResult {
+    parsed: number;
+    warnings: string[];
+    stages: Stages;
+    messages: string[];
+}
 export interface Builder {
-    build(forcedPaths?: ReadonlySet<string>): Promise<{
-        parsed: number;
-        warnings: string[];
-        stages: Stages;
-    }>;
-    recover(): Promise<{
-        parsed: number;
-        warnings: string[];
-        stages: Stages;
-    }>;
-    invalidate(kind: EmbedChangeKind): Promise<{
-        parsed: number;
-    }>;
-    invalidateFeatures(toggles: FeatureToggle[]): Promise<{
-        parsed: number;
-    }>;
+    build(): Promise<BuildResult>;
     close(): Promise<void>;
     readonly poolsCreated: number;
 }
-export declare function createBuilder(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect): Builder;
+type EnsureFeatureSchema = (cfg: ResolvedConfig) => Promise<void>;
+export declare function createBuilder(conn: Connection, cfg: ResolvedConfig, dialect: ReconcileDialect, ensureFeatureSchema: EnsureFeatureSchema): Builder;
+export {};