sparkforensics-mcp 0.1.0

package/bin/sparkforensics-mcp.mjs

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+// bin/sparkforensics-mcp.mjs
+import { existsSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+
+const binDir = dirname(fileURLToPath(import.meta.url));
+const pkgDir = dirname(binDir);
+
+// vendor-core/ exists only in a published/standalone install (populated by
+// scripts/vendor-core.mjs at pack time); in the monorepo it falls back to
+// the real packages/core/src/ sibling. load-vendored.js itself is the one
+// module this bootstrap has to locate by hand (see its own header comment);
+// every other core module then loads through its exported loadVendored().
+async function loadCreateMcpServer() {
+  const vendoredHelper = join(pkgDir, 'vendor-core', 'load-vendored.js');
+  const helperPath = existsSync(vendoredHelper) ? vendoredHelper : join(pkgDir, '..', 'core', 'src', 'load-vendored.js');
+  const { loadVendored } = await import(pathToFileURL(helperPath).href);
+  const mod = await loadVendored(pkgDir, 'mcp-server-factory');
+  return mod.createMcpServer;
+}
+
+async function main() {
+  const createMcpServer = await loadCreateMcpServer();
+  const server = createMcpServer();
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}
+
+await main();

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "sparkforensics-mcp",
+  "version": "0.1.0",
+  "mcpName": "io.github.shuffle-works/sparkforensics-mcp",
+  "type": "module",
+  "license": "MIT",
+  "description": "MCP server exposing Apache Spark event-log diagnostics (diagnose_run, get_run_summary, compare_runs, get_finding_evidence, evaluate_budgets) over stdio.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shuffle-works/sparkforensics.git"
+  },
+  "bin": {
+    "sparkforensics-mcp": "./bin/sparkforensics-mcp.mjs"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "files": ["bin/", "vendor-core/"],
+  "scripts": {
+    "prepack": "node ../../scripts/vendor-core.mjs .",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "zod": "^4.4.3"
+  },
+  "devDependencies": {
+    "vitest": "^4.1.10"
+  }
+}

package/vendor-core/analyzer.js

@@ -0,0 +1,167 @@
+import { DETECTORS,               } from './detectors.js';
+import { computePeakConcurrentCores } from './core-count.js';
+import { assertNever } from './assert-never.js';
+import { estimateImpact } from './impact-estimator.js';
+import { computeOccupancy,                     } from './occupancy.js';
+import { deriveImpactBand } from './impact-band.js';
+import { IMPACT_BAND_ORDER } from './format-utils.js';
+             
+                                                                                
+                    
+
+// FNV-1a 32-bit: a small, dependency-free stable string hash. Used to derive
+// a deterministic finding `id` at the single choke point below so the same
+// logical finding keeps the same id across runs (no timestamps, no randomness).
+function fnv1a(str        )         {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(16).padStart(8, '0');
+}
+
+export function findingId(f         )         {
+  // Location key mirrors the report's stable identity: stage, else SQL
+  // execution, else the audited config property.
+  const locKey = f.stageId ?? f.executionId ?? f.property ?? '';
+  // Discriminators for detectors that intentionally emit multiple findings on
+  // the same location+metric; without them these siblings hash to one id:
+  //   stage-scope: slowHost (host), memoryUtilization (executorId + dimension),
+  //     partitionSizing (rule);
+  //   app-scope: cacheUtilization (rddId + variant, stageId is always null),
+  //     cachingOpportunity (stageId/executionId both null too: `relation`/
+  //     `format` distinguish leaf findings, `operator`+`relation` distinguish
+  //     composite findings, `executionIds` as a last resort when everything
+  //     else matches);
+  //   SQL plan-advisor: smallFiles (direction/nodeName),
+  //     duplicatePlanSubtree (rootName/subtreeSize/groupIndex: two unrelated
+  //     duplicate-subtree groups can share rootName+subtreeSize, e.g. the same
+  //     "BroadcastExchange over a Project/Filter/Scan" shape repeated once per
+  //     dimension table; groupIndex (the group's deterministic position within
+  //     one execution's plan) is the actual uniqueness guarantee, since the
+  //     informational `sampleRelation` field can be null or coincide for two
+  //     groups),
+  //     broadcastSizing (per node/side, distinguished by value + largerSideBytes).
+  // The metric `value` folds in the per-node magnitude that broadcastSizing
+  // siblings carry no other field for; it is deterministic for a given log, so
+  // the id stays stable across re-parses (no timestamp/randomness).
+  // memoryUtilization's `rule` (heapNearCapacity/heapOverProvisioned) is excluded here: an
+  // executor falls in exactly one heap band, so `executorId` alone already guarantees
+  // uniqueness and folding `rule` in too would just add gratuitous id-rotation risk if the
+  // band logic ever changes. partitionSizing's `rule` (maxPartitionTooBig/
+  // shufflePartitionSkew/lowShuffleParallelism) IS load-bearing here: a stage can emit more
+  // than one of those rules at once, sharing the same stageId+metric.
+  const rule = f.type === 'memoryUtilization' ? undefined : f.rule;
+  const disc = [
+    f.host, f.executorId, rule, f.variant, f.dimension,
+    f.direction, f.nodeName, f.rootName, f.subtreeSize, f.groupIndex, f.largerSideBytes,
+    f.rddId, f.relation, f.format, f.operator,
+    f.executionIds ? f.executionIds.join(',') : '',
+  ].map((v) => v ?? '').join('|');
+  return fnv1a(`${f.type}|${locKey}|${f.metric ?? ''}|${f.value ?? ''}|${disc}`);
+}
+
+function push(out           , entry          , result                            )       {
+  if (!result) return;
+  const detectorVersion = entry.version ?? 1;
+  for (const f of (Array.isArray(result) ? result : [result])) {
+    if (!f) continue;
+    if (entry.suppressWhen && entry.suppressWhen(f, out)) continue;
+    const stamped = { ...f, docAnchor: f.docAnchor ?? entry.docAnchor, detectorVersion };
+    const id = findingId(stamped);
+    // Guardrail against any detector (this one or a future one) emitting two
+    // structurally-identical findings for what should be one logical
+    // occurrence: same id => same finding, keep only the first. Added after a
+    // real-log bug where duplicatePlanSubtree emitted two distinct findings
+    // sharing one id (see findingId's groupIndex discriminator above and
+    // tests/detectors-plan.test.js's duplicatePlanSubtree regression tests);
+    // this guard's correctness depends on findingId's discriminator list
+    // actually being unique per distinct finding, not on this line itself.
+    if (out.some((existing) => existing.id === id)) continue;
+    out.push({ ...stamped, id });
+  }
+}
+
+// `app` is widened to `SparkAppInfo | null` rather than the plan's literal
+// non-nullable `SparkAppInfo`: real callers (evidence-report.ts's
+// `buildJson`, useIngest.ts's `runDone`/`snapshotParsedRun`) pass
+// `AppModel.app`, which is genuinely `SparkAppInfo | null` at the type
+// level even though by the time `analyze()` runs in practice a full parse
+// has completed and `app` is always populated (the same reasoning
+// detectors.ts's `DetectorCtx.app` comment gives for treating its own,
+// separate `app` field as non-nullable). Widened to match the real caller
+// type rather than forcing a cast at every call site.
+export function analyze(
+  app                     ,
+  stages                    ,
+  executorsAdded                 ,
+  executorsRemoved                 ,
+  jobs                  ,
+  sql                            = new Map(),
+  runAggregates                       = null,
+)            {
+  // `app ?? {}`: every detector below tolerates a null `app` (malformed logs
+  // with no ApplicationStart), so this call must too; computePeakConcurrentCores
+  // just falls back to the executor-derived core sum when `resources` is absent.
+  // computePeakConcurrentCores (not computeTotalCores) here specifically: the
+  // ceiling this feeds (src/occupancy.ts's computeCeiling) needs a genuine
+  // concurrent-capacity bound, and computeTotalCores's cumulative sum over
+  // every addition overstates that under dynamic allocation/executor
+  // replacement (a churned-through executor's cores were never actually
+  // concurrent with its replacement's).
+  // `executorsAdded`/`executorsRemoved` casts: computePeakConcurrentCores only
+  // reads `executorId`/`timestamp`/`totalCores`, all present on
+  // ExecutorAddedEvent/ExecutorRemovedEvent, but `totalCores` isn't on
+  // ExecutorRemovedEvent, so the `ExecutorEvent` union as a whole is a
+  // structural mismatch against its parameter shapes.
+  const totalCores = computePeakConcurrentCores(
+    app ?? {},
+    executorsAdded                                                                         ,
+    executorsRemoved                                                    ,
+  );
+  // Computed once, up front, so detectors can gate impact band on the same
+  // occupancy-clipped waste figure estimateImpact() below displays as that
+  // finding's savings (src/detectors.ts's `clippedWasteMs`), not a raw
+  // pre-clip delta the two passes would otherwise disagree on.
+  const occupancy = computeOccupancy(stages                                          , totalCores);
+  const ctx = {
+    app, stages, executorsAdded, executorsRemoved, jobs, sql, runAggregates, occupancy,
+  };
+  const out            = [];
+  for (const d of DETECTORS) {
+    if (d.inScorecard === false) continue;
+    switch (d.scope) {
+      case 'stage':
+        for (const s of stages.values()) push(out, d, d.detect(s, ctx));
+        break;
+      case 'sql':
+        for (const e of sql.values()) push(out, d, d.detect(e, ctx));
+        break;
+      case 'app':
+      case 'config':
+        push(out, d, d.detect(ctx));
+        break;
+      default:
+        assertNever(d.scope);
+    }
+  }
+  estimateImpact(out, stages, totalCores);
+  deriveImpactBand(out, app);
+  // Ascending IMPACT_BAND_ORDER (critical 0 → info 2) puts the worst band
+  // first, matching the old descending 3/2/1 rank this replaced; `sort` is
+  // stable, so findings sharing a band keep their DETECTORS declaration order.
+  out.sort((a, b) => IMPACT_BAND_ORDER[a.impactBand] - IMPACT_BAND_ORDER[b.impactBand]);
+  return out;
+}
+
+export function auditConfig(app                     )            {
+  const out            = [];
+  for (const d of DETECTORS) if (d.scope === 'config') push(out, d, d.detect({ app }));
+  // configAudit's own impact-estimator case (src/impact-estimator.ts) needs neither
+  // `stages` nor `totalCores`: it's unconditionally `costOnly('none')`. An empty stages
+  // map is enough for parity with the same finding type produced via analyze().
+  estimateImpact(out, new Map());
+  deriveImpactBand(out, app);
+  return out.map((f) => ({ ...f, stageId: f.stageId ?? null }));
+}

package/vendor-core/assert-never.js

@@ -0,0 +1,3 @@
+export function assertNever(x       )        {
+  throw new Error(`Unreachable case: ${JSON.stringify(x)}`);
+}

package/vendor-core/cli/budgets.js

@@ -0,0 +1,203 @@
+import { computeEfficiencyModel } from '../efficiency-model.js';
+import { computeSkewRatio, DETECTORS } from '../detectors.js';
+import { IMPACT_BAND_ORDER } from '../format-utils.js';
+                                                              
+                                                                 
+
+const IMPACT_BANDS = Object.keys(IMPACT_BAND_ORDER)                ;
+
+                                
+                                                                    
+                                                           
+                                                                                  
+ 
+                               
+                                                                                            
+                                              
+                                                
+                 
+ 
+
+const skewDetector = DETECTORS.find((d) => d.type === 'skew');
+const SKEW_MIN_TASKS_FOR_P95 = skewDetector .thresholds .minTasksForP95          ;
+
+function taskDataTrusted(appModel          )          {
+  const entry = appModel.evidenceAvailability?.entries?.find((e) => e.key === 'taskCoreTime');
+  return entry?.state === 'present';
+}
+
+// `Finding.value` is `number | string` (some detectors, e.g. stageFailed/
+// configAudit, put human-readable text there instead of a magnitude; see
+// types.ts). The 'spill' findings this is used for are always numeric; the
+// typeof guard below narrows without changing behavior for real input.
+function maxFindingValue(catalog           , type        )                {
+  const values = catalog
+    .filter((f) => f.type === type)
+    .map((f) => f.value ?? 0)
+    .filter((v)              => typeof v === 'number');
+  return values.length > 0 ? Math.max(...values) : null;
+}
+
+function checkRuntime(appModel          , maxRuntimeMs        )               {
+  const { startTime, endTime } = appModel.app ?? {};
+  if (startTime == null || endTime == null) {
+    return { name: 'max-runtime', status: 'inconclusive', detail: 'App start/end time not observed (run may not have finished).' };
+  }
+  const runtimeMs = endTime - startTime;
+  return runtimeMs > maxRuntimeMs
+    ? { name: 'max-runtime', status: 'violation', detail: `Runtime ${runtimeMs}ms exceeds budget ${maxRuntimeMs}ms.` }
+    : { name: 'max-runtime', status: 'pass', detail: `Runtime ${runtimeMs}ms within budget ${maxRuntimeMs}ms.` };
+}
+
+function checkSpill(appModel          , catalog           , maxSpillGb        )               {
+  if (!taskDataTrusted(appModel)) {
+    return { name: 'max-spill', status: 'inconclusive', detail: 'No trustworthy task-level evidence to measure spill.' };
+  }
+  const maxBytes = maxFindingValue(catalog, 'spill') ?? 0;
+  const budgetBytes = maxSpillGb * 1024 ** 3;
+  return maxBytes > budgetBytes
+    ? { name: 'max-spill', status: 'violation', detail: `Peak stage spill ${maxBytes} bytes exceeds budget ${budgetBytes} bytes.` }
+    : { name: 'max-spill', status: 'pass', detail: `Peak stage spill ${maxBytes} bytes within budget ${budgetBytes} bytes.` };
+}
+
+function checkSkew(appModel          , maxSkewRatio        )               {
+  if (!taskDataTrusted(appModel)) {
+    return { name: 'max-skew', status: 'inconclusive', detail: 'No trustworthy task-level evidence to measure skew.' };
+  }
+  const stages = [...(appModel.stages?.values() ?? [])];
+  if (stages.length === 0) {
+    return { name: 'max-skew', status: 'inconclusive', detail: 'No stage data observed in this event log.' };
+  }
+  // Recompute the true ratio per stage (not from the finding catalog): the
+  // skew detector floors its findings at thresholds.ratioWarn (3), so a
+  // budget stricter than that floor could never be enforced by reading
+  // catalog findings alone.
+  const ratios = stages
+    .map((stage) => computeSkewRatio(stage, SKEW_MIN_TASKS_FOR_P95))
+    .filter((r) => r !== null)
+    .map((r) => r.ratio);
+  if (ratios.length === 0) {
+    return { name: 'max-skew', status: 'inconclusive', detail: 'No stage has a measurable task-duration median.' };
+  }
+  const maxRatio = Math.max(...ratios);
+  return maxRatio > maxSkewRatio
+    ? { name: 'max-skew', status: 'violation', detail: `Peak stage skew ratio ${maxRatio} exceeds budget ${maxSkewRatio}.` }
+    : { name: 'max-skew', status: 'pass', detail: `Peak stage skew ratio ${maxRatio} within budget ${maxSkewRatio}.` };
+}
+
+function checkFailedTaskRate(appModel          , catalog           , maxPct        )               {
+  const finding = catalog.find((f) => f.type === 'jobFailureRate');
+  if (finding) {
+    const rate = (finding.taskFailureRate                      ) ?? 0;
+    return rate > maxPct
+      ? { name: 'max-failed-task-rate', status: 'violation', detail: `Task failure rate ${rate}% exceeds budget ${maxPct}%.` }
+      : { name: 'max-failed-task-rate', status: 'pass', detail: `Task failure rate ${rate}% within budget ${maxPct}%.` };
+  }
+  if ((appModel.jobs?.size ?? 0) === 0) {
+    return { name: 'max-failed-task-rate', status: 'inconclusive', detail: 'No job data observed in this event log.' };
+  }
+  // Jobs completed but the jobFailureRate detector never fired: job failure
+  // rate is below its own 10% info floor (src/detectors.js), so the task
+  // failure rate is implicitly low too. Known v1 limitation: a budget
+  // stricter than that floor cannot be enforced.
+  return { name: 'max-failed-task-rate', status: 'pass', detail: `No job-failure-rate finding: task failure rate is below the detector's reporting floor.` };
+}
+
+function checkEfficiency(appModel          , minPct        )               {
+  if (!taskDataTrusted(appModel)) {
+    return { name: 'min-efficiency', status: 'inconclusive', detail: 'No trustworthy task-level evidence to measure efficiency.' };
+  }
+  const model = computeEfficiencyModel({
+    app: appModel.app, stages: appModel.stages,
+    executorsAdded: appModel.executors.added, runAggregates: appModel.runAggregates,
+  });
+  if (model.wastagePct == null) {
+    return { name: 'min-efficiency', status: 'inconclusive', detail: 'Efficiency could not be computed (no available compute hours).' };
+  }
+  const efficiencyPct = 100 - model.wastagePct;
+  return efficiencyPct < minPct
+    ? { name: 'min-efficiency', status: 'violation', detail: `Efficiency ${efficiencyPct}% below budget ${minPct}%.` }
+    : { name: 'min-efficiency', status: 'pass', detail: `Efficiency ${efficiencyPct}% meets budget ${minPct}%.` };
+}
+
+function checkRegression(comparison                   , maxRegressionPct        , regressionMetric        )               {
+  const row = comparison.metrics.find((m) => m.key === regressionMetric);
+  if (!row || row.direction === 'unavailable' || row.baseline == null || row.delta == null) {
+    return { name: 'max-regression', status: 'inconclusive', detail: `Metric "${regressionMetric}" is unavailable for this comparison.` };
+  }
+  // A neutral-direction metric (inputBytes, outputBytes, taskCount,
+  // executorsAdded — see NEUTRAL_METRIC_KEYS in run-comparison.ts) measures
+  // workload volume, not performance: an increase isn't a regression, so a
+  // regression budget can't be meaningfully evaluated against it either way.
+  if (row.direction === 'neutral') {
+    return { name: 'max-regression', status: 'inconclusive', detail: `Metric "${regressionMetric}" measures workload volume, not performance: it has no regression direction to check.` };
+  }
+  if (row.direction !== 'regression') {
+    return { name: 'max-regression', status: 'pass', detail: `Metric "${regressionMetric}" did not regress (${row.direction}).` };
+  }
+  const pct = row.baseline === 0 ? Infinity : Math.abs(row.delta / row.baseline) * 100;
+  const pctLabel = row.baseline === 0
+    ? `regressed from 0 to ${row.delta} (was absent/zero in baseline)`
+    : `regressed ${pct.toFixed(1)}%`;
+  return pct > maxRegressionPct
+    ? { name: 'max-regression', status: 'violation', detail: `Metric "${regressionMetric}" ${pctLabel}, exceeding budget ${maxRegressionPct}%.` }
+    : { name: 'max-regression', status: 'pass', detail: `Metric "${regressionMetric}" ${pctLabel}, within budget ${maxRegressionPct}%.` };
+}
+
+function checkFailOnIntroduced(comparison                   , band        )               {
+  if (band !== 'all' && !IMPACT_BANDS.includes(band              )) {
+    return { name: 'fail-on-introduced', status: 'inconclusive', detail: `Impact band "${band}" is not recognized (expected "all" or one of ${IMPACT_BANDS.join(', ')}).` };
+  }
+  const matches = band === 'all'
+    ? comparison.findings.introduced
+    : comparison.findings.introduced.filter((f) => f.impactBand === band);
+  return matches.length > 0
+    ? { name: 'fail-on-introduced', status: 'violation', detail: `${matches.length} introduced finding(s) match "${band}".` }
+    : { name: 'fail-on-introduced', status: 'pass', detail: `No introduced findings match "${band}".` };
+}
+
+// Shared by both comparison-dependent budgets below: each needs the same
+// "no comparison yet -> inconclusive" fallback instead of actually checking.
+function pushComparisonBudget(
+  results                ,
+  comparison                               ,
+  name                      ,
+  check                                                 ,
+)       {
+  results.push(comparison ? check(comparison) : { name, status: 'inconclusive', detail: 'No baseline comparison available to evaluate this budget.' });
+}
+
+export function evaluateBudgets({ appModel, catalog, budgets, comparison }   
+                                                                                                 
+ )                                                                        {
+  const results                 = [];
+  if (Number.isFinite(budgets.maxRuntimeMs)) results.push(checkRuntime(appModel, budgets.maxRuntimeMs ));
+  if (Number.isFinite(budgets.maxSpillGb)) results.push(checkSpill(appModel, catalog, budgets.maxSpillGb ));
+  if (Number.isFinite(budgets.maxSkewRatio)) results.push(checkSkew(appModel, budgets.maxSkewRatio ));
+  if (Number.isFinite(budgets.maxFailedTaskRatePct)) results.push(checkFailedTaskRate(appModel, catalog, budgets.maxFailedTaskRatePct ));
+  if (Number.isFinite(budgets.minEfficiencyPct)) results.push(checkEfficiency(appModel, budgets.minEfficiencyPct ));
+  // Guarded here (not just at the CLI/mcp-tools call sites) so any caller of
+  // evaluateBudgets() gets this for free: regressionMetric with no
+  // maxRegressionPct would otherwise skip the whole `if` below silently,
+  // reporting nothing at all instead of a visible inconclusive result.
+  if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
+    results.push({ name: 'max-regression', status: 'inconclusive', detail: `regressionMetric "${budgets.regressionMetric}" was set without maxRegressionPct; the regression budget was not evaluated.` });
+  } else if (budgets.maxRegressionPct !== undefined) {
+    // `!== undefined`, not `Number.isFinite`: a zero-baseline regression's pct
+    // is itself `Infinity` (see checkRegression), so an "unlimited" budget is a
+    // legitimate finite-typed-as-number input here (CLI's own flag validation
+    // already rejects non-finite --max-regression-pct input, so this only
+    // widens what a direct evaluateBudgets caller, e.g. a test, can express).
+    pushComparisonBudget(results, comparison, 'max-regression',
+      (c) => checkRegression(c, budgets.maxRegressionPct , budgets.regressionMetric ?? 'wallClock'));
+  }
+  if (budgets.failOnIntroduced !== undefined) {
+    pushComparisonBudget(results, comparison, 'fail-on-introduced',
+      (c) => checkFailOnIntroduced(c, budgets.failOnIntroduced ));
+  }
+  return {
+    results,
+    violated: results.some((r) => r.status === 'violation'),
+    inconclusive: results.some((r) => r.status === 'inconclusive'),
+  };
+}

package/vendor-core/cli/collect-run.js

@@ -0,0 +1,107 @@
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, basename } from 'node:path';
+import { createState, runParse, runParseFiles, reassembleRollingEntries } from '../parser-worker.js';
+import { createModelCallbacks } from '../model-assembler.js';
+import { routeMessage,                     } from '../ingest.js';
+                                            
+
+                           
+                             
+                                                                             
+                                      
+ 
+
+export function emptyAppModel()           {
+  return {
+    app: null,
+    stages: new Map(),
+    executors: { added: [], removed: [] },
+    sql: new Map(),
+    jobs: new Map(),
+    runAggregates: null,
+    evidenceAvailability: null,
+  };
+}
+
+// Mirrors tests/parser-worker.test.js's fakeFile: the proven File-like shape
+// runParse/runParseFiles need (name, size, slice().arrayBuffer(), arrayBuffer()).
+export function nodeFileFromPath(path        )           {
+  const bytes = readFileSync(path);
+  const u8 = new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  return {
+    name: basename(path),
+    size: u8.length,
+    slice(start        , end        ) {
+      const view = u8.subarray(start, end);
+      return { async arrayBuffer() { return view.slice().buffer; } };
+    },
+    async arrayBuffer() { return u8.slice().buffer; },
+  };
+}
+
+// Delegates to src/ingest.js's routeMessage: the single source of truth for
+// worker-message-type -> handler-callback wiring, shared by the browser
+// Worker context and this synchronous-in-Node CLI path. No pendingTaskRequests
+// map: the CLI never sends a 'getTaskData' request, so it never receives a
+// 'taskData' message back.
+export function dispatch(msg         , handlers                )       {
+  routeMessage(msg                                          , handlers);
+}
+
+function isRollingLogDirectory(dirPath        )          {
+  const names = readdirSync(dirPath);
+  return names.some((n        ) => /^events_\d+_/.test(n));
+}
+
+// Shared ingest scaffold for both consumers of routeMessage's onDone/onError
+// dispatch pattern (this file's collectRun, over a local file/dir, and
+// shs-load.ts's collectShsAppModel, over fetched archive bytes): builds a
+// fresh AppModel wired to the shared handlers, then hands `run` a
+// (state, emit, reject) triple so each caller only supplies its own decode
+// step (and its own error type via `onDecodeError` — a plain Error here, an
+// mcpError over in shs-load.ts).
+export function collectViaDispatch(
+  run                                                                                                                   ,
+  onDecodeError                         ,
+)                                                        {
+  const appModel = emptyAppModel();
+  const cb = createModelCallbacks(appModel, { onProgress() {}, onDone() {}, onError() {} });
+
+  return new Promise((resolve, reject) => {
+    const handlers                 = {
+      ...cb,
+      onDone: (msg         ) => resolve({ appModel, skippedLines: (msg                             )?.skippedLines ?? 0 }),
+      onError: (msg         ) => reject(onDecodeError(msg)),
+    };
+    const emit = (msg         ) => dispatch(msg, handlers);
+    const state = createState();
+    run(state, emit, reject);
+  });
+}
+
+export async function collectRun(inputPath        )                                                        {
+  const stat = statSync(inputPath);
+  return collectViaDispatch((state, emit, reject) => {
+    if (stat.isDirectory()) {
+      if (!isRollingLogDirectory(inputPath)) {
+        reject(new Error("This isn't a Spark rolling event-log directory. Pass a single event-log file instead."));
+        return;
+      }
+      const names = readdirSync(inputPath);
+      let ordered;
+      try {
+        ordered = reassembleRollingEntries(names);
+      } catch (e) {
+        reject(e);
+        return;
+      }
+      const files = ordered.map((name) => nodeFileFromPath(join(inputPath, name)));
+      // .catch(reject), not void: an unexpected throw past the parser's own
+      // guards (e.g. in decoder.flush) would otherwise leave this Promise
+      // pending forever and surface only as an unhandled rejection.
+      runParseFiles(files, state, { emit }).catch(reject);
+    } else {
+      runParse(nodeFileFromPath(inputPath), state, { emit }).catch(reject);
+    }
+  }, (msg) => new Error((msg                       ).message));
+}

package/vendor-core/core-count.js

@@ -0,0 +1,68 @@
+// Total cores computation: sum real executor totalCores, or fallback to
+// peakExecutors × configured cores. Used by efficiency model, scaling simulator,
+// wasted core-hours calculation, and utilization/memory-utilization detectors.
+export function computeTotalCores(app                                             , executorsAdded                              )         {
+  let total = executorsAdded.reduce((s, e) => s + (e.totalCores ?? 0), 0);
+  if (total <= 0) {
+    const cores = app.resources?.executor?.cores ?? null;
+    total = cores != null ? executorsAdded.length * cores : 0;
+  }
+  return total;
+}
+
+// Peak concurrently-alive core count: sweeps add/remove events by timestamp
+// (add contributes +totalCores at its own executorId, remove contributes
+// -totalCores looked up from that same executorId) and tracks the running
+// total's max, rather than summing every addition regardless of overlap.
+// computeTotalCores's cumulative sum overstates capacity under dynamic
+// allocation/executor replacement, since a churned-through executor's cores
+// are never actually concurrent with its replacement's; this is the
+// concurrency-aware sibling used to bound the impact estimator's ceiling
+// (src/occupancy.ts's computeCeiling) against that inflation.
+// Tie-break same-timestamp events by delta ascending, so a removal is applied
+// before a same-instant replacement's addition: without this, a seamless swap
+// (old executor gone exactly when its replacement joins) would momentarily
+// double-count both as concurrent.
+function sweepPeak(events                                      )         {
+  const sorted = [...events].sort((a, b) => a.time - b.time || a.delta - b.delta);
+  let running = 0;
+  let peak = 0;
+  for (const ev of sorted) {
+    running += ev.delta;
+    if (running > peak) peak = running;
+  }
+  return peak;
+}
+
+export function computePeakConcurrentCores(
+  app                                             ,
+  executorsAdded                                                                     ,
+  executorsRemoved                                                ,
+)         {
+  const coresByExecutor = new Map                ();
+  const coreEvents                                       = [];
+  const countEvents                                       = [];
+  for (const e of executorsAdded) {
+    const cores = e.totalCores ?? 0;
+    coresByExecutor.set(e.executorId, cores);
+    coreEvents.push({ time: e.timestamp, delta: cores });
+    countEvents.push({ time: e.timestamp, delta: 1 });
+  }
+  for (const e of executorsRemoved) {
+    const cores = coresByExecutor.get(e.executorId) ?? 0;
+    coreEvents.push({ time: e.timestamp, delta: -cores });
+    countEvents.push({ time: e.timestamp, delta: -1 });
+  }
+  const peak = sweepPeak(coreEvents);
+  if (peak > 0) return peak;
+  // Real totalCores data is missing/zero for every add event, so the cores
+  // sweep above can't tell us anything. Falling back to
+  // `executorsAdded.length * cores` here would reintroduce the exact
+  // cumulative-historical-additions overcount this function exists to avoid
+  // (a churned-through executor and its replacement both counted, even
+  // though they were never alive at once). Sweep peak *executor count*
+  // instead: still concurrency-aware, just cores-blind.
+  const peakExecutorCount = sweepPeak(countEvents);
+  const cores = app.resources?.executor?.cores ?? null;
+  return cores != null ? peakExecutorCount * cores : 0;
+}