sparkforensics-cli 0.1.0 → 0.2.2

package/vendor-core/analyzer.js

@@ -9,9 +9,7 @@ import { IMPACT_BAND_ORDER } from './format-utils.js';
                                                                                 
                     
 
-// FNV-1a 32-bit: a small, dependency-free stable string hash. Used to derive
-// a deterministic finding `id` at the single choke point below so the same
-// logical finding keeps the same id across runs (no timestamps, no randomness).
+// FNV-1a 32-bit stable string hash: deterministic finding id across runs, no timestamps/randomness.
 function fnv1a(str        )         {
   let h = 0x811c9dc5;
   for (let i = 0; i < str.length; i++) {
@@ -22,36 +20,19 @@ function fnv1a(str        )         {
 }
 
 export function findingId(f         )         {
-  // Location key mirrors the report's stable identity: stage, else SQL
-  // execution, else the audited config property.
+  // Location key: stage, else SQL execution, else audited config property.
   const locKey = f.stageId ?? f.executionId ?? f.property ?? '';
-  // Discriminators for detectors that intentionally emit multiple findings on
-  // the same location+metric; without them these siblings hash to one id:
-  //   stage-scope: slowHost (host), memoryUtilization (executorId + dimension),
-  //     partitionSizing (rule);
-  //   app-scope: cacheUtilization (rddId + variant, stageId is always null),
-  //     cachingOpportunity (stageId/executionId both null too: `relation`/
-  //     `format` distinguish leaf findings, `operator`+`relation` distinguish
-  //     composite findings, `executionIds` as a last resort when everything
-  //     else matches);
-  //   SQL plan-advisor: smallFiles (direction/nodeName),
-  //     duplicatePlanSubtree (rootName/subtreeSize/groupIndex: two unrelated
-  //     duplicate-subtree groups can share rootName+subtreeSize, e.g. the same
-  //     "BroadcastExchange over a Project/Filter/Scan" shape repeated once per
-  //     dimension table; groupIndex (the group's deterministic position within
-  //     one execution's plan) is the actual uniqueness guarantee, since the
-  //     informational `sampleRelation` field can be null or coincide for two
-  //     groups),
-  //     broadcastSizing (per node/side, distinguished by value + largerSideBytes).
-  // The metric `value` folds in the per-node magnitude that broadcastSizing
-  // siblings carry no other field for; it is deterministic for a given log, so
-  // the id stays stable across re-parses (no timestamp/randomness).
-  // memoryUtilization's `rule` (heapNearCapacity/heapOverProvisioned) is excluded here: an
-  // executor falls in exactly one heap band, so `executorId` alone already guarantees
-  // uniqueness and folding `rule` in too would just add gratuitous id-rotation risk if the
-  // band logic ever changes. partitionSizing's `rule` (maxPartitionTooBig/
-  // shufflePartitionSkew/lowShuffleParallelism) IS load-bearing here: a stage can emit more
-  // than one of those rules at once, sharing the same stageId+metric.
+  // Discriminators for detectors that emit multiple findings on the same
+  // location+metric; without them these siblings hash to one id:
+  //   slowHost (host), memoryUtilization (executorId+dimension), partitionSizing (rule);
+  //   cacheUtilization (rddId+variant), cachingOpportunity (relation/format for leaf,
+  //     operator+relation for composite, executionIds as last resort);
+  //   smallFiles (direction/nodeName), duplicatePlanSubtree (groupIndex is the real
+  //     uniqueness guarantee: rootName+subtreeSize can collide across groups),
+  //     broadcastSizing (value+largerSideBytes per node/side).
+  // memoryUtilization excludes `rule`: one heap band per executor, so executorId is already
+  // unique and folding rule in risks id churn if band logic changes. partitionSizing keeps
+  // `rule`: a stage can emit several rules at once sharing stageId+metric.
   const rule = f.type === 'memoryUtilization' ? undefined : f.rule;
   const disc = [
     f.host, f.executorId, rule, f.variant, f.dimension,
@@ -62,6 +43,35 @@ export function findingId(f         )         {
   return fnv1a(`${f.type}|${locKey}|${f.metric ?? ''}|${f.value ?? ''}|${disc}`);
 }
 
+// skew's max/median branch (stage.taskCount below minTasksForP95) and straggler are both driven
+// by the identical (taskDurationMax - taskDurationP50) delta on the same stage: the same
+// dominant outlier task reported by two detectors, each independently clipped (see "Overlap
+// caveat: skew / straggler" in impact-estimation.md). skew's P95/median branch samples a
+// different task and stays independent. Flags both sides via validationRequired (rather than
+// suppressing either) so neither finding's own diagnostic value is lost; the flag rides the same
+// confidence-caveat UI a reader already sees before trusting either finding's magnitude.
+function overlapNote(otherType                      )         {
+  return `This overlaps with the ${otherType} finding on this stage: both are driven by the same dominant outlier task, so don't add their recoverable-time figures together.`;
+}
+
+function flagSkewStragglerOverlap(findings           )       {
+  const maxMedianSkewStages = new Set(
+    findings.filter((f) => f.type === 'skew' && f.metric === 'max/median' && f.stageId != null).map((f) => f.stageId),
+  );
+  if (maxMedianSkewStages.size === 0) return;
+  const stragglerStages = new Set(
+    findings.filter((f) => f.type === 'straggler' && f.stageId != null).map((f) => f.stageId),
+  );
+  const overlapStages = new Set([...maxMedianSkewStages].filter((id) => stragglerStages.has(id)));
+  if (overlapStages.size === 0) return;
+  for (const f of findings) {
+    if (f.stageId == null || !overlapStages.has(f.stageId)) continue;
+    const note = f.type === 'skew' ? overlapNote('straggler') : f.type === 'straggler' ? overlapNote('skew') : null;
+    if (!note) continue;
+    f.validationRequired = f.validationRequired ? `${f.validationRequired} ${note}` : note;
+  }
+}
+
 function push(out           , entry          , result                            )       {
   if (!result) return;
   const detectorVersion = entry.version ?? 1;
@@ -70,28 +80,15 @@ function push(out           , entry          , result
     if (entry.suppressWhen && entry.suppressWhen(f, out)) continue;
     const stamped = { ...f, docAnchor: f.docAnchor ?? entry.docAnchor, detectorVersion };
     const id = findingId(stamped);
-    // Guardrail against any detector (this one or a future one) emitting two
-    // structurally-identical findings for what should be one logical
-    // occurrence: same id => same finding, keep only the first. Added after a
-    // real-log bug where duplicatePlanSubtree emitted two distinct findings
-    // sharing one id (see findingId's groupIndex discriminator above and
-    // tests/detectors-plan.test.js's duplicatePlanSubtree regression tests);
-    // this guard's correctness depends on findingId's discriminator list
-    // actually being unique per distinct finding, not on this line itself.
+    // Dedup guard: same id => same finding, keep first. Correctness depends on
+    // findingId's discriminators being unique per distinct finding, not on this line.
     if (out.some((existing) => existing.id === id)) continue;
     out.push({ ...stamped, id });
   }
 }
 
-// `app` is widened to `SparkAppInfo | null` rather than the plan's literal
-// non-nullable `SparkAppInfo`: real callers (evidence-report.ts's
-// `buildJson`, useIngest.ts's `runDone`/`snapshotParsedRun`) pass
-// `AppModel.app`, which is genuinely `SparkAppInfo | null` at the type
-// level even though by the time `analyze()` runs in practice a full parse
-// has completed and `app` is always populated (the same reasoning
-// detectors.ts's `DetectorCtx.app` comment gives for treating its own,
-// separate `app` field as non-nullable). Widened to match the real caller
-// type rather than forcing a cast at every call site.
+// `app` widened to `SparkAppInfo | null` to match real callers (AppModel.app is
+// nullable at the type level); every detector below tolerates a null app.
 export function analyze(
   app                     ,
   stages                    ,
@@ -101,29 +98,18 @@ export function analyze(
   sql                            = new Map(),
   runAggregates                       = null,
 )            {
-  // `app ?? {}`: every detector below tolerates a null `app` (malformed logs
-  // with no ApplicationStart), so this call must too; computePeakConcurrentCores
-  // just falls back to the executor-derived core sum when `resources` is absent.
-  // computePeakConcurrentCores (not computeTotalCores) here specifically: the
-  // ceiling this feeds (src/occupancy.ts's computeCeiling) needs a genuine
-  // concurrent-capacity bound, and computeTotalCores's cumulative sum over
-  // every addition overstates that under dynamic allocation/executor
-  // replacement (a churned-through executor's cores were never actually
-  // concurrent with its replacement's).
-  // `executorsAdded`/`executorsRemoved` casts: computePeakConcurrentCores only
-  // reads `executorId`/`timestamp`/`totalCores`, all present on
-  // ExecutorAddedEvent/ExecutorRemovedEvent, but `totalCores` isn't on
-  // ExecutorRemovedEvent, so the `ExecutorEvent` union as a whole is a
-  // structural mismatch against its parameter shapes.
+  // `app ?? {}`: detectors tolerate a null app (malformed logs), so this must too.
+  // computePeakConcurrentCores (not computeTotalCores): the occupancy ceiling needs a
+  // concurrent-capacity bound; a cumulative sum overstates it under dynamic allocation.
+  // Casts: the function reads only executorId/timestamp/totalCores, but totalCores isn't
+  // on ExecutorRemovedEvent, so the ExecutorEvent union mismatches its parameter shapes.
   const totalCores = computePeakConcurrentCores(
     app ?? {},
     executorsAdded                                                                         ,
     executorsRemoved                                                    ,
   );
-  // Computed once, up front, so detectors can gate impact band on the same
-  // occupancy-clipped waste figure estimateImpact() below displays as that
-  // finding's savings (src/detectors.ts's `clippedWasteMs`), not a raw
-  // pre-clip delta the two passes would otherwise disagree on.
+  // Computed once so detectors gate impact band on the same occupancy-clipped waste
+  // estimateImpact displays as savings, not a raw pre-clip delta the two passes would disagree on.
   const occupancy = computeOccupancy(stages                                          , totalCores);
   const ctx = {
     app, stages, executorsAdded, executorsRemoved, jobs, sql, runAggregates, occupancy,
@@ -148,20 +134,34 @@ export function analyze(
   }
   estimateImpact(out, stages, totalCores);
   deriveImpactBand(out, app);
-  // Ascending IMPACT_BAND_ORDER (critical 0 → info 2) puts the worst band
-  // first, matching the old descending 3/2/1 rank this replaced; `sort` is
-  // stable, so findings sharing a band keep their DETECTORS declaration order.
+  flagSkewStragglerOverlap(out);
+  // Ascending IMPACT_BAND_ORDER (critical 0 -> info 2) puts the worst band first;
+  // stable sort keeps DETECTORS declaration order within a band.
   out.sort((a, b) => IMPACT_BAND_ORDER[a.impactBand] - IMPACT_BAND_ORDER[b.impactBand]);
   return out;
 }
 
-export function auditConfig(app                     )            {
+// Memoizes auditConfig by `app` identity (like evidence-report.ts's jsonCache) so config detectors
+// don't re-run when both the export and the report path audit the same app. WeakMap can't key on
+// `null`, so that case skips the cache. Returns a fresh copy each call so a caller's in-place
+// mutation (e.g. `.sort()`) can't corrupt the cached array.
+const auditConfigCache = new WeakMap                         ();
+
+function computeAuditConfig(app                     )            {
   const out            = [];
   for (const d of DETECTORS) if (d.scope === 'config') push(out, d, d.detect({ app }));
-  // configAudit's own impact-estimator case (src/impact-estimator.ts) needs neither
-  // `stages` nor `totalCores`: it's unconditionally `costOnly('none')`. An empty stages
-  // map is enough for parity with the same finding type produced via analyze().
+  // configAudit's impact case is unconditionally costOnly('none'): needs no stages/totalCores,
+  // an empty stages map gives parity with analyze().
   estimateImpact(out, new Map());
   deriveImpactBand(out, app);
   return out.map((f) => ({ ...f, stageId: f.stageId ?? null }));
 }
+
+export function auditConfig(app                     )            {
+  if (app === null) return computeAuditConfig(app);
+  const cached = auditConfigCache.get(app);
+  if (cached) return cached.slice();
+  const result = computeAuditConfig(app);
+  auditConfigCache.set(app, result);
+  return result.slice();
+}

package/vendor-core/cli/budgets.js

@@ -26,10 +26,8 @@ function taskDataTrusted(appModel          )          {
   return entry?.state === 'present';
 }
 
-// `Finding.value` is `number | string` (some detectors, e.g. stageFailed/
-// configAudit, put human-readable text there instead of a magnitude; see
-// types.ts). The 'spill' findings this is used for are always numeric; the
-// typeof guard below narrows without changing behavior for real input.
+// Finding.value is number|string (some detectors put text there); spill findings are
+// always numeric, so the typeof guard narrows without changing behavior for real input.
 function maxFindingValue(catalog           , type        )                {
   const values = catalog
     .filter((f) => f.type === type)
@@ -68,10 +66,8 @@ function checkSkew(appModel          , maxSkewRatio        )               {
   if (stages.length === 0) {
     return { name: 'max-skew', status: 'inconclusive', detail: 'No stage data observed in this event log.' };
   }
-  // Recompute the true ratio per stage (not from the finding catalog): the
-  // skew detector floors its findings at thresholds.ratioWarn (3), so a
-  // budget stricter than that floor could never be enforced by reading
-  // catalog findings alone.
+  // Recompute the true ratio per stage: the skew detector floors findings at
+  // thresholds.ratioWarn (3), so a stricter budget can't be enforced from catalog alone.
   const ratios = stages
     .map((stage) => computeSkewRatio(stage, SKEW_MIN_TASKS_FOR_P95))
     .filter((r) => r !== null)
@@ -96,10 +92,8 @@ function checkFailedTaskRate(appModel          , catalog           , maxPct
   if ((appModel.jobs?.size ?? 0) === 0) {
     return { name: 'max-failed-task-rate', status: 'inconclusive', detail: 'No job data observed in this event log.' };
   }
-  // Jobs completed but the jobFailureRate detector never fired: job failure
-  // rate is below its own 10% info floor (src/detectors.js), so the task
-  // failure rate is implicitly low too. Known v1 limitation: a budget
-  // stricter than that floor cannot be enforced.
+  // jobFailureRate never fired => rate below its 10% info floor, so task failure rate is
+  // implicitly low. v1 limitation: a budget stricter than that floor can't be enforced.
   return { name: 'max-failed-task-rate', status: 'pass', detail: `No job-failure-rate finding: task failure rate is below the detector's reporting floor.` };
 }
 
@@ -125,10 +119,8 @@ function checkRegression(comparison                   , maxRegressionPct
   if (!row || row.direction === 'unavailable' || row.baseline == null || row.delta == null) {
     return { name: 'max-regression', status: 'inconclusive', detail: `Metric "${regressionMetric}" is unavailable for this comparison.` };
   }
-  // A neutral-direction metric (inputBytes, outputBytes, taskCount,
-  // executorsAdded — see NEUTRAL_METRIC_KEYS in run-comparison.ts) measures
-  // workload volume, not performance: an increase isn't a regression, so a
-  // regression budget can't be meaningfully evaluated against it either way.
+  // A neutral-direction metric (inputBytes/outputBytes/taskCount/executorsAdded) measures
+  // workload volume, not performance: an increase isn't a regression, so no direction to check.
   if (row.direction === 'neutral') {
     return { name: 'max-regression', status: 'inconclusive', detail: `Metric "${regressionMetric}" measures workload volume, not performance: it has no regression direction to check.` };
   }
@@ -156,8 +148,7 @@ function checkFailOnIntroduced(comparison                   , band        )
     : { name: 'fail-on-introduced', status: 'pass', detail: `No introduced findings match "${band}".` };
 }
 
-// Shared by both comparison-dependent budgets below: each needs the same
-// "no comparison yet -> inconclusive" fallback instead of actually checking.
+// Both comparison-dependent budgets share the "no comparison yet -> inconclusive" fallback.
 function pushComparisonBudget(
   results                ,
   comparison                               ,
@@ -176,18 +167,13 @@ export function evaluateBudgets({ appModel, catalog, budgets, comparison }
   if (Number.isFinite(budgets.maxSkewRatio)) results.push(checkSkew(appModel, budgets.maxSkewRatio ));
   if (Number.isFinite(budgets.maxFailedTaskRatePct)) results.push(checkFailedTaskRate(appModel, catalog, budgets.maxFailedTaskRatePct ));
   if (Number.isFinite(budgets.minEfficiencyPct)) results.push(checkEfficiency(appModel, budgets.minEfficiencyPct ));
-  // Guarded here (not just at the CLI/mcp-tools call sites) so any caller of
-  // evaluateBudgets() gets this for free: regressionMetric with no
-  // maxRegressionPct would otherwise skip the whole `if` below silently,
-  // reporting nothing at all instead of a visible inconclusive result.
+  // Guarded here so any evaluateBudgets caller benefits: regressionMetric without
+  // maxRegressionPct would otherwise skip the `if` silently, reporting nothing.
   if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
     results.push({ name: 'max-regression', status: 'inconclusive', detail: `regressionMetric "${budgets.regressionMetric}" was set without maxRegressionPct; the regression budget was not evaluated.` });
   } else if (budgets.maxRegressionPct !== undefined) {
-    // `!== undefined`, not `Number.isFinite`: a zero-baseline regression's pct
-    // is itself `Infinity` (see checkRegression), so an "unlimited" budget is a
-    // legitimate finite-typed-as-number input here (CLI's own flag validation
-    // already rejects non-finite --max-regression-pct input, so this only
-    // widens what a direct evaluateBudgets caller, e.g. a test, can express).
+    // `!== undefined`, not Number.isFinite: a zero-baseline regression's pct is Infinity,
+    // so an "unlimited" budget is a legitimate input (the CLI already rejects non-finite flags).
     pushComparisonBudget(results, comparison, 'max-regression',
       (c) => checkRegression(c, budgets.maxRegressionPct , budgets.regressionMetric ?? 'wallClock'));
   }

package/vendor-core/cli/collect-run.js

@@ -1,4 +1,4 @@
-import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { readFileSync, readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
 import { join, basename } from 'node:path';
 import { createState, runParse, runParseFiles, reassembleRollingEntries } from '../parser-worker.js';
 import { createModelCallbacks } from '../model-assembler.js';
@@ -23,8 +23,7 @@ export function emptyAppModel()           {
   };
 }
 
-// Mirrors tests/parser-worker.test.js's fakeFile: the proven File-like shape
-// runParse/runParseFiles need (name, size, slice().arrayBuffer(), arrayBuffer()).
+// File-like shape runParse/runParseFiles need: name, size, slice().arrayBuffer(), arrayBuffer().
 export function nodeFileFromPath(path        )           {
   const bytes = readFileSync(path);
   const u8 = new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
@@ -39,27 +38,53 @@ export function nodeFileFromPath(path        )           {
   };
 }
 
-// Delegates to src/ingest.js's routeMessage: the single source of truth for
-// worker-message-type -> handler-callback wiring, shared by the browser
-// Worker context and this synchronous-in-Node CLI path. No pendingTaskRequests
-// map: the CLI never sends a 'getTaskData' request, so it never receives a
-// 'taskData' message back.
+// Lazy-loading variant for peekLogHeader: only reads the requested byte range from disk,
+// avoiding eager full-file slurps for large logs during directory scans.
+// Used by list_runs local-mode scanning; does NOT have arrayBuffer() since peekLogHeader
+// only needs slice() for bounded reads of the file header.
+export function nodeLazyPeekableFromPath(path        )   
+               
+               
+                                                                             
+  {
+  const size = statSync(path).size;
+  return {
+    name: basename(path),
+    size,
+    slice(start        , end        ) {
+      return {
+        async arrayBuffer()                       {
+          const len = Math.max(0, Math.min(end, size) - start);
+          if (len === 0) return new ArrayBuffer(0);
+          const buf = Buffer.alloc(len);
+          const fd = openSync(path, 'r');
+          try {
+            readSync(fd, buf, 0, len, start);
+          } finally {
+            closeSync(fd);
+          }
+          return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+        },
+      };
+    },
+  };
+}
+
+// routeMessage is the single source of truth for worker-message -> handler wiring, shared by
+// the browser Worker and this Node CLI path. No pendingTaskRequests: the CLI never sends
+// 'getTaskData', so never receives 'taskData' back.
 export function dispatch(msg         , handlers                )       {
   routeMessage(msg                                          , handlers);
 }
 
-function isRollingLogDirectory(dirPath        )          {
+export function isRollingLogDirectory(dirPath        )          {
   const names = readdirSync(dirPath);
   return names.some((n        ) => /^events_\d+_/.test(n));
 }
 
-// Shared ingest scaffold for both consumers of routeMessage's onDone/onError
-// dispatch pattern (this file's collectRun, over a local file/dir, and
-// shs-load.ts's collectShsAppModel, over fetched archive bytes): builds a
-// fresh AppModel wired to the shared handlers, then hands `run` a
-// (state, emit, reject) triple so each caller only supplies its own decode
-// step (and its own error type via `onDecodeError` — a plain Error here, an
-// mcpError over in shs-load.ts).
+// Shared ingest scaffold for both routeMessage consumers (this file's collectRun over a
+// local file/dir, shs-load.ts's collectShsAppModel over fetched bytes): each caller supplies
+// its own decode step and error type via onDecodeError.
 export function collectViaDispatch(
   run                                                                                                                   ,
   onDecodeError                         ,
@@ -96,9 +121,8 @@ export async function collectRun(inputPath        )
         return;
       }
       const files = ordered.map((name) => nodeFileFromPath(join(inputPath, name)));
-      // .catch(reject), not void: an unexpected throw past the parser's own
-      // guards (e.g. in decoder.flush) would otherwise leave this Promise
-      // pending forever and surface only as an unhandled rejection.
+      // .catch(reject), not void: a throw past the parser's guards would otherwise leave
+      // this Promise pending forever, surfacing only as an unhandled rejection.
       runParseFiles(files, state, { emit }).catch(reject);
     } else {
       runParse(nodeFileFromPath(inputPath), state, { emit }).catch(reject);

package/vendor-core/core-count.js

@@ -1,6 +1,4 @@
-// Total cores computation: sum real executor totalCores, or fallback to
-// peakExecutors × configured cores. Used by efficiency model, scaling simulator,
-// wasted core-hours calculation, and utilization/memory-utilization detectors.
+// Total cores: sum executor totalCores, else peakExecutors x configured cores.
 export function computeTotalCores(app                                             , executorsAdded                              )         {
   let total = executorsAdded.reduce((s, e) => s + (e.totalCores ?? 0), 0);
   if (total <= 0) {
@@ -10,19 +8,11 @@ export function computeTotalCores(app
   return total;
 }
 
-// Peak concurrently-alive core count: sweeps add/remove events by timestamp
-// (add contributes +totalCores at its own executorId, remove contributes
-// -totalCores looked up from that same executorId) and tracks the running
-// total's max, rather than summing every addition regardless of overlap.
-// computeTotalCores's cumulative sum overstates capacity under dynamic
-// allocation/executor replacement, since a churned-through executor's cores
-// are never actually concurrent with its replacement's; this is the
-// concurrency-aware sibling used to bound the impact estimator's ceiling
-// (src/occupancy.ts's computeCeiling) against that inflation.
-// Tie-break same-timestamp events by delta ascending, so a removal is applied
-// before a same-instant replacement's addition: without this, a seamless swap
-// (old executor gone exactly when its replacement joins) would momentarily
-// double-count both as concurrent.
+// Peak concurrently-alive core count: sweep add/remove events by timestamp, track running max,
+// rather than summing every addition. A cumulative sum overstates capacity under dynamic
+// allocation (a churned-through executor's cores were never concurrent with its replacement's).
+// Tie-break same-timestamp events by delta ascending so a removal applies before a same-instant
+// addition; else a seamless swap would momentarily double-count both as concurrent.
 function sweepPeak(events                                      )         {
   const sorted = [...events].sort((a, b) => a.time - b.time || a.delta - b.delta);
   let running = 0;
@@ -41,28 +31,36 @@ export function computePeakConcurrentCores(
 )         {
   const coresByExecutor = new Map                ();
   const coreEvents                                       = [];
-  const countEvents                                       = [];
   for (const e of executorsAdded) {
     const cores = e.totalCores ?? 0;
     coresByExecutor.set(e.executorId, cores);
     coreEvents.push({ time: e.timestamp, delta: cores });
-    countEvents.push({ time: e.timestamp, delta: 1 });
   }
   for (const e of executorsRemoved) {
     const cores = coresByExecutor.get(e.executorId) ?? 0;
     coreEvents.push({ time: e.timestamp, delta: -cores });
-    countEvents.push({ time: e.timestamp, delta: -1 });
   }
   const peak = sweepPeak(coreEvents);
   if (peak > 0) return peak;
-  // Real totalCores data is missing/zero for every add event, so the cores
-  // sweep above can't tell us anything. Falling back to
-  // `executorsAdded.length * cores` here would reintroduce the exact
-  // cumulative-historical-additions overcount this function exists to avoid
-  // (a churned-through executor and its replacement both counted, even
-  // though they were never alive at once). Sweep peak *executor count*
-  // instead: still concurrency-aware, just cores-blind.
-  const peakExecutorCount = sweepPeak(countEvents);
+  // totalCores missing/zero for every add, so the cores sweep is blind. Falling back to
+  // executorsAdded.length * cores would reintroduce the cumulative overcount this avoids;
+  // sweep peak executor count instead: concurrency-aware, cores-blind.
+  const peakExecutorCount = computePeakConcurrentExecutorCount(executorsAdded, executorsRemoved);
   const cores = app.resources?.executor?.cores ?? null;
   return cores != null ? peakExecutorCount * cores : 0;
 }
+
+// Peak concurrently-alive executor COUNT (not cores): the denominator memoryUtilization's
+// per-executor waste math (allocatedMB x executor count) needs. Same sweep as
+// computePeakConcurrentCores, over add/remove events rather than cores, so a churned-through
+// executor's slot is never double-counted against its replacement's.
+export function computePeakConcurrentExecutorCount(
+  executorsAdded                                                ,
+  executorsRemoved                                                ,
+)         {
+  const countEvents                                       = [
+    ...executorsAdded.map((e) => ({ time: e.timestamp, delta: 1 })),
+    ...executorsRemoved.map((e) => ({ time: e.timestamp, delta: -1 })),
+  ];
+  return sweepPeak(countEvents);
+}

package/vendor-core/core-locality-ratio.js

@@ -1,14 +1,7 @@
-// Whole-run core-usage-locality ratio: non-local task share across every
-// stage's `stage.localityStats`. Mirrors wasted-core-hours.js's shape (pure
-// reducer, no detector or view coupling), importable by both the
-// `coreLocality` detector (src/detectors.js) and CoreUsageArea.tsx's
-// stage-breakdown list.
-//
-// Locality classification (Spark, best to worst): PROCESS_LOCAL > NODE_LOCAL
-// > NO_PREF > RACK_LOCAL > ANY. NO_PREF is not a locality failure; it's what
-// shuffle-read stages report because there's no location-preference concept
-// for a shuffle fetch, so it stays in the denominator (diluting the ratio for
-// shuffle-heavy stages, which is correct) but never in the numerator.
+// Whole-run core-usage-locality ratio: non-local task share across every stage's localityStats.
+// Locality tiers (Spark, best to worst): PROCESS_LOCAL > NODE_LOCAL > NO_PREF > RACK_LOCAL > ANY.
+// NO_PREF is not a locality failure (shuffle-read stages report it, no location preference for a
+// shuffle fetch), so it stays in the denominator but never the numerator.
 const NON_LOCAL_TIERS = new Set(['RACK_LOCAL', 'ANY']);
 const TOP_N = 5;
 const MIN_TASKS_PER_STAGE = 10;

package/vendor-core/core-time-series.js

@@ -1,11 +1,7 @@
-// Sweep-line "busy cores over time" computation. Pure and side-effect-free so
-// it can run inside parser-worker.js (over retained task launch/finish
-// timestamps) or on the main thread (scaling simulator). One core per task:
-// Spark's default task-to-core mapping.
-//
-// `hypotheticalCores` clamps the concurrent busy-core count to N (the
-// deterministic "utilization at N cores" curve). It does NOT re-schedule tasks;
-// makespan re-estimation is the scaling simulator's job, layered on this signal.
+// Sweep-line "busy cores over time". Pure so it runs in parser-worker (over retained task
+// timestamps) or on the main thread (scaling simulator). One core per task: Spark's default.
+// `hypotheticalCores` clamps the concurrent busy-core count to N; it does NOT re-schedule
+// tasks (makespan re-estimation is the scaling simulator's job, layered on this signal).
 
                     
                  
@@ -19,18 +15,16 @@
  
 
 function buildStepFunction(intervals            , hypotheticalCores               )            {
-  // Emit +1 at launch, -1 at finish. Skip empty/negative intervals.
   const events                                      = [];
   for (const { launch, finish } of intervals) {
     if (!(finish > launch)) continue;
     events.push({ t: launch, delta: 1 });
     events.push({ t: finish, delta: -1 });
   }
-  // Sort by time; at equal time process -1 before +1 so [launch, finish) is
-  // half-open (a task finishing exactly as another launches does not overlap).
+  // At equal time process -1 before +1 so [launch, finish) is half-open (a task finishing
+  // exactly as another launches does not overlap).
   events.sort((a, b) => (a.t - b.t) || (a.delta - b.delta));
 
-  // Walk consecutive timestamps, emitting the busy level held on each interval.
   const segments            = [];
   let busy = 0;
   for (let i = 0; i < events.length; i++) {

package/vendor-core/core-usage-locality.js

@@ -1,7 +1,6 @@
-// Stage-granular approximation of core-usage-by-locality over time. See the
-// plan's Task 6 DEVIATION note: per-task locality is not retained, so this
-// distributes each stage's executorRunTime across its wall-clock window,
-// split by localityStats proportions. Not exact; approximate by construction.
+// Stage-granular approximation of core-usage-by-locality over time. Per-task locality is not
+// retained, so this distributes each stage's executorRunTime across its wall-clock window,
+// split by localityStats proportions. Approximate by construction.
 export const LOCALITY_TIERS           = ['PROCESS_LOCAL', 'NODE_LOCAL', 'RACK_LOCAL', 'NO_PREF', 'ANY'];