sparkforensics-cli 0.1.0

package/vendor-core/format-utils.js

@@ -0,0 +1,167 @@
+const TARGET_PARTITION_BYTES = 128 * 1024 * 1024;
+const MAX_RECOMMENDED = 8000;
+const MEANINGFUL_RATIO = 1.5;
+
+export const VISIBLE_LIMIT         = 6;
+export const IMPACT_BAND_ORDER                                                  = { critical: 0, warning: 1, info: 2 };
+
+const BOTTLENECK_WIDGET                         = {
+  skew: 'task-skew', slowHost: 'task-skew', straggler: 'task-skew',
+  shuffle: 'shuffle-io', spill: 'spill', gc: 'gc-pressure', failures: 'failures',
+  coldStart: 'executor-timeline', utilization: 'executor-timeline', speculationWaste: 'executor-timeline',
+};
+
+// Cross-widget stage recurrence: how many *distinct board widgets* (not raw
+// catalog entries: skew+straggler on one stage still count as one widget,
+// task-skew) flag a given stage. Computed once from the full catalog each
+// widget already receives, so no new parameter threads through the
+// dashboard-renderer render loop.
+// `stageId` widened to also accept `undefined` (Task 22, src/detectors.ts):
+// sql/config-scope findings never set `stageId` at all, so `Finding.stageId`
+// is now `number | null | undefined`, and this `catalog` param is usually a
+// live `Finding[]`. `stageId == null` below already treats both the same.
+export function stageWidgetFrequency(catalog                                                )                      {
+  const perStage = new Map();
+  for (const b of catalog) {
+    if (b.stageId == null) continue;
+    const widget = BOTTLENECK_WIDGET[b.type];
+    if (!widget) continue;
+    if (!perStage.has(b.stageId)) perStage.set(b.stageId, new Set());
+    perStage.get(b.stageId).add(widget);
+  }
+  const freq = new Map();
+  for (const [stageId, widgets] of perStage) freq.set(stageId, widgets.size);
+  return freq;
+}
+
+// Canonical detector type → ALL-CAPS board tag vocabulary (AGENTS.md "Problem
+// flagging"). Single source of truth for every widget that needs to render a
+// catalog entry's tag outside its own dedicated board card, e.g. Bottleneck
+// Alerts, which lists findings across every detector type.
+// Exported (not just used internally by typeTag) so doc-sync checks can
+// enumerate every tag value without hand-duplicating this list elsewhere.
+export const TYPE_TAG_MAP                         = {
+  skew: 'SKEW', shuffle: 'SHFL', spill: 'SPILL', gc: 'GC',
+  coldStart: 'COLD', utilization: 'UTIL', memoryUtilization: 'MEM',
+  cacheUtilization: 'CSTOR', coreLocality: 'LOCAL', autoscalingChurn: 'CHRN',
+  slowHost: 'HOST', failures: 'FAIL', straggler: 'STRAG',
+  retryWaste: 'RETRY', tinyTask: 'TINY', stageFailed: 'SFAIL',
+  speculationWaste: 'SPEC',
+  partitionSizing: 'PART', stageSlowness: 'SLOW', stageShape: 'SHAPE',
+  cachingOpportunity: 'CACHE', jobFailureRate: 'JOBS', configAudit: 'CFG',
+  duplicatePlanSubtree: 'PLAN', smallFiles: 'PLAN', underBroadcast: 'PLAN', overBroadcast: 'PLAN',
+  broadcastSizing: 'PLAN',
+  incompleteRun: 'INCMP',
+};
+
+export function typeTag(type        )         {
+  return TYPE_TAG_MAP[type] ?? type.toUpperCase();
+}
+
+// Human-readable tooltips for the terse spill-classification badges
+// (skew | vol | ?), surfaced via title= so the vocabulary is decipherable
+// without an external key. Kept domain-agnostic (Spark-internal terms only).
+export const SPILL_CLASS_TITLE = {
+  skew: 'Skew spill: a few heavy tasks spill while most do not; rebalance partitioning',
+  volume: 'Volume spill: most tasks spill because data exceeds memory; add partitions',
+  unclassified: 'Spill cause could not be classified',
+};
+
+/**
+ * Worst (lowest IMPACT_BAND_ORDER) impact band across a list of findings,
+ * `undefined` when the list is empty: shared by every widget that needs a
+ * single band to badge/color a group of findings (Topbar, ShuffleIO,
+ * GcPressure, TaskSkew, Failures, ConfigAudit, Alerts).
+ * @param {{ impactBand: 'critical' | 'warning' | 'info' }[]} findings
+ * @returns {'critical' | 'warning' | 'info' | undefined}
+ */
+export function worstImpactBand(
+  findings                                                   ,
+)                                              {
+  let worst;
+  for (const f of findings) {
+    if (worst === undefined || IMPACT_BAND_ORDER[f.impactBand] < IMPACT_BAND_ORDER[worst]) worst = f.impactBand;
+  }
+  return worst;
+}
+
+export function escHtml(str         )         {
+  return String(str).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+// Reduces an embedded HDFS/S3/absolute-path fragment in a string to its
+// basename (e.g. "Scan ExistingRDD ... - hdfs://host/a/b/c" -> "c"). Strings
+// with no such fragment pass through unchanged; this is NOT a general
+// truncator. Shared by cache-utilization.js (RDD names), detectors.js (plan
+// node names embedded in recommendation text), and stage-detail.js (plan
+// node detail labels).
+export function pathBasename(str         )         {
+  const s = String(str);
+  const m = s.match(/((?:hdfs?|s3[an]?):\/\/\S+|\/[^\s,)]+)/i);
+  if (!m) return s;
+  const segs = m[1].replace(/[/,]+$/, '').split('/').filter(Boolean);
+  return segs[segs.length - 1] || s;
+}
+
+// "collect at /u02/hadoop/.../utils.py:1869" -> "collect at utils.py:1869". Splits
+// on the last " at " (Spark's own callsite format is "<operation> at <location>"),
+// then reduces the location to its basename via pathBasename (whose match already
+// includes a trailing ":<line>", since colons aren't excluded from its regex).
+export function trimCallsite(callsite        )         {
+  const idx = callsite.lastIndexOf(' at ');
+  if (idx === -1) return callsite;
+  const operation = callsite.slice(0, idx);
+  const location = callsite.slice(idx + 4);
+  return `${operation} at ${pathBasename(location)}`;
+}
+
+// recent-files ids are `${name}::${size}::${lastModified}`: show just the
+// name. Preferred over `app.name`, which is usually identical across the two
+// runs being compared and so wouldn't distinguish baseline from candidate.
+export function runLabel(id        )         {
+  return id.split('::')[0] || id;
+}
+
+export function formatBytes(bytes                    )         {
+  if (!bytes || bytes <= 0) return '—';
+  if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
+  if (bytes >= 1e6) return `${(bytes / 1e6).toFixed(0)} MB`;
+  return `${(bytes / 1e3).toFixed(0)} KB`;
+}
+
+// Spark reports some duration fields (e.g. executorCpuTime) in nanoseconds
+// while most of this codebase works in milliseconds; convert at the source
+// rather than trusting call sites to remember the unit.
+export function nsToMs(ns        )         {
+  return ns / 1e6;
+}
+
+export function formatDuration(ms                    )         {
+  if (!ms || ms <= 0) return '—';
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+  return `${Math.floor(ms / 60000)}m ${Math.floor((ms % 60000) / 1000)}s`;
+}
+
+export function recommendPartitions(stage                                                 )                                                {
+  const bytes = stage.shuffleReadBytes ?? 0;
+  if (bytes <= 0) return null;
+  const recommended = Math.min(MAX_RECOMMENDED, Math.ceil(bytes / TARGET_PARTITION_BYTES));
+  const current = stage.taskCount ?? 0;
+  if (recommended <= current * MEANINGFUL_RATIO) return null;
+  return { recommended, current };
+}
+
+export function buildHistogram(values          , bins        )                                     {
+  if (values.length === 0) return { labels: [], data: [] };
+  let min = values[0], max = values[0];
+  for (let i = 1; i < values.length; i++) {
+    if (values[i] < min) min = values[i];
+    if (values[i] > max) max = values[i];
+  }
+  const binSize = (max - min) / bins || 1;
+  const counts = new Array(bins).fill(0);
+  for (const v of values) counts[Math.min(Math.floor((v - min) / binSize), bins - 1)]++;
+  const labels = counts.map((_, i) => Math.round(min + i * binSize));
+  return { labels, data: counts };
+}

package/vendor-core/impact-band.js

@@ -0,0 +1,50 @@
+                                                        
+import { STRAGGLER_FLOOR_PCT_WARN, STRAGGLER_FLOOR_PCT_CRIT } from './detectors.js';
+
+// Reused from `straggler`'s own thresholds (detectors.ts's
+// `floorPctWarn`/`floorPctCrit`, marked `NOT SOURCED: unvalidated` there
+// already): applying the same, already-accepted noise floor globally rather
+// than inventing a second, unrelated cutoff. Imported (not re-literaled) so
+// the two can't drift out of sync.
+const IMPACT_FLOOR_PCT_WARN = STRAGGLER_FLOOR_PCT_WARN;
+const IMPACT_FLOOR_PCT_CRIT = STRAGGLER_FLOOR_PCT_CRIT;
+
+function appDurationMs(app                     )                {
+  if (app?.startTime == null || app?.endTime == null) return null;
+  const durationMs = app.endTime - app.startTime;
+  return durationMs > 0 ? durationMs : null;
+}
+
+/**
+ * Overwrites `.impactBand` for any finding with a quantified wall-clock
+ * estimate, grading it purely by recoverable time as a fraction of the
+ * run's total duration: a full replace (can move a finding's fixed fallback
+ * band in either direction once real recoverable time is known), not a
+ * promote-only floor. Findings with no `wallClock` estimate
+ * (`resourceOnly`/`informational` basis) or an unknown/zero app duration are
+ * left exactly as the detector set them: there is no comparable figure to
+ * grade them by, so the detector's own fixed classification stands as the
+ * final value. Mutates in place and returns the same array, matching
+ * `estimateImpact`'s own signature style.
+ *
+ * Grades `wallClock.high`, matching `triage-target.ts`/`FixTheseFirst.tsx`/
+ * `formatImpactEstimateCompact`, which already rank and display severity by
+ * the same optimistic figure. This is a deliberate split from
+ * `view/impact-sort.ts`'s `'impact'` sort mode, which ranks by `.low`
+ * instead: severity grading (this function) wants the best-case number a fix
+ * could realistically claim, while the sort wants the guaranteed floor so an
+ * optimistic-but-contended finding never outranks a smaller but certain one.
+ * Same range, two different fields for two different questions — not an
+ * inconsistency to reconcile.
+ */
+export function deriveImpactBand(findings           , app                     )            {
+  const durationMs = appDurationMs(app);
+  if (durationMs == null) return findings;
+  for (const finding of findings) {
+    const recoverableMs = finding.impactEstimate?.wallClock?.high;
+    if (recoverableMs == null) continue;
+    const pct = recoverableMs / durationMs;
+    finding.impactBand = pct >= IMPACT_FLOOR_PCT_CRIT ? 'critical' : pct >= IMPACT_FLOOR_PCT_WARN ? 'warning' : 'info';
+  }
+  return findings;
+}

package/vendor-core/impact-estimator.js

@@ -0,0 +1,428 @@
+                                                                                                       
+import { computeOccupancy, estimateSingleStage, estimateMultiStage,                                              } from './occupancy.js';
+import { detectorCatalog } from './detectors.js';
+
+// Assumed shuffle-network throughput, ~1 Gbps. Starting assumption; tune against the
+// Task 10 real-log spot-check (docs/architecture.md's Impact estimation section).
+const SHUFFLE_THROUGHPUT_BPS = 125_000_000;
+// Assumed disk I/O throughput for spilled data, ~200 MB/s (conservative HDD/SSD blend).
+const SPILL_IO_THROUGHPUT_BPS = 200_000_000;
+// Spark's classic recommended shuffle partition size.
+const IDEAL_BYTES_PER_PARTITION_TASK = 128 * 1024 * 1024;
+// Assumed per-task scheduling/launch overhead.
+const TASK_SCHEDULING_OVERHEAD_MS = 50;
+// Assumed per-file open latency (small-file overhead).
+const FILE_OPEN_OVERHEAD_MS = 10;
+// Assumed broadcast-transfer bandwidth, shared with overBroadcast/underBroadcast.
+const BROADCAST_BANDWIDTH_BPS = 125_000_000;
+// Assumed per-non-local-task network-fetch penalty, reported as extra core-time.
+const NETWORK_FETCH_PENALTY_MS = 20;
+// Assumed executor JVM+container startup overhead.
+const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
+// Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
+const RE_READ_THROUGHPUT_BPS = 125_000_000;
+
+// The stageSlowness detector (src/detectors.ts) flags a stage once its wall-clock
+// duration reaches `infoMin` minutes; that's the point beyond which the stage's time
+// stops being "normal", so it's also the floor for the waste this estimate reports.
+// Read from the detector's own catalog entry (not a duplicated literal) so the two
+// stay in sync automatically if the detector's threshold ever changes.
+const STAGE_SLOWNESS_THRESHOLD_MINUTES = (() => {
+  const infoMin = detectorCatalog().find((d) => d.type === 'stageSlowness')?.thresholds?.infoMin;
+  if (typeof infoMin !== 'number') {
+    throw new Error("impact-estimator: stageSlowness detector's 'infoMin' threshold not found in detectorCatalog()");
+  }
+  return infoMin;
+})();
+
+// A finding with no quantifiable magnitude falls back to 'informational'; one that
+// still yields a rawWaste figure (independent of any stage window) falls back to
+// 'resourceOnly'. Never a fake {low: 0, high: 0}: wallClock is null in both cases.
+function costOnly(estimateMethod                      , rawWaste                 )                 {
+  return rawWaste
+    ? { basis: 'resourceOnly', wallClock: null, estimateMethod, rawWaste }
+    : { basis: 'informational', wallClock: null, estimateMethod };
+}
+
+function singleStageImpact(
+  wasteMs        ,
+  stageId        ,
+  stages                    ,
+  occupancy                                 ,
+  estimateMethod                      ,
+  rawWaste                 ,
+)                 {
+  const est = estimateSingleStage(wasteMs, stageId, stages                                          , occupancy);
+  if (est) return { basis: est.basis, wallClock: est.wallClock, estimateMethod, rawWaste };
+  return costOnly(estimateMethod, rawWaste); // stage excluded from the sweep (duration <= 0)
+}
+
+function stageMappableWasteOrCostOnly(
+  wasteMs        ,
+  stageIds                      ,
+  stages                    ,
+  occupancy                                 ,
+)                 {
+  const rawWaste                             = wasteMs > 0 ? { value: wasteMs, unit: 'ms' } : undefined;
+  if (!stageIds || stageIds.length === 0) {
+    return costOnly('modeled', rawWaste);
+  }
+  // One waste event spread over a span of stages, not N independent wastes:
+  // apportion it evenly so estimateMultiStage's union cap doesn't have to
+  // absorb the same amount claimed once per stage.
+  const perStageWasteMs = wasteMs / stageIds.length;
+  const wasteMsByStage = new Map(stageIds.map((id) => [id, perStageWasteMs]));
+  const est = estimateMultiStage(stageIds, wasteMsByStage, stages                                          , occupancy);
+  if (!est) return costOnly('modeled', rawWaste); // every stage excluded from the sweep
+  return { basis: est.basis, wallClock: est.wallClock, estimateMethod: 'modeled', rawWaste };
+}
+
+/**
+ * Per-finding-type dispatch, populated incrementally: each formula task adds one
+ * case here. A type with no case stays uncovered (no impactEstimate attached),
+ * every type this design covers must eventually get one, per the Global Constraints'
+ * "no silent omissions" rule; docs/architecture.md's Impact estimation table is the
+ * authoritative completeness check, not this switch by itself.
+ */
+function computeEstimateForFinding(
+  finding         ,
+  stages                    ,
+  occupancy                                 ,
+)                        {
+  switch (finding.type) {
+    case 'retryWaste': {
+      // The waste figure lives on the Stage, not the Finding: the detector only
+      // re-publishes it as `metric`/`value` (src/detectors.ts's retryWaste entry).
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const wasteMs = (stage.retryWasteMs                      ) ?? 0;
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+    }
+    case 'speculationWaste': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const wasteMs = (stage.speculationWasteMs                      ) ?? 0;
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+    }
+    case 'coldStart': {
+      // The detector reports the gap as `metric: 'startupGapSeconds', value: <seconds>`.
+      if (typeof finding.value !== 'number') return null;
+      const wasteMs = finding.value * 1000;
+      // Time before any task starts can never overlap any stage; a genuine,
+      // unclipped point estimate, not tied to any stage's own gate (coldStart is
+      // app-scoped, stageId: null): see Non-goals in the original design spec.
+      return { basis: 'serial', wallClock: { low: wasteMs, high: wasteMs }, estimateMethod: 'measured' };
+    }
+    case 'gc': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+      const executorRunTime = stage.executorRunTime ?? 0;
+      const jvmGCTime = stage.jvmGCTime ?? 0;
+      // The raw cross-task core-time sum, before any conversion: the one figure here
+      // that is straight from the log rather than modeled.
+      const rawWaste = { value: jvmGCTime, unit: 'coreMs' }         ;
+      if (executorRunTime <= 0 || stageDurationMs <= 0) {
+        return costOnly('modeled', rawWaste);
+      }
+      const avgConcurrency = executorRunTime / stageDurationMs;
+      // jvmGCTime is a cross-task core-time sum, the same aggregation shape as
+      // executorRunTime; dividing by the stage's own average concurrency converts it
+      // to an approximate wall-clock figure. This rides on measured inputs but is a
+      // modeled approximation, not an exact reconstruction: hence 'modeled', and see
+      // docs/architecture.md's spot-check note next to this formula.
+      const wasteMs = jvmGCTime / avgConcurrency;
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', rawWaste);
+    }
+    case 'skew': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const p50 = stage.taskDurationP50 ?? 0;
+      // computeSkewRatio's own metric labels (src/detectors.ts): 'P95/median' or 'max/median'.
+      const usesP95Branch = finding.metric === 'P95/median';
+      const wasteMs = Math.max(0, usesP95Branch ? (stage.taskDurationP95 ?? 0) - p50 : (stage.taskDurationMax ?? 0) - p50);
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+    }
+    case 'straggler':
+    case 'stageShape': {
+      // NOTE: this is a shared case for two finding types. straggler (which has
+      // no `rule` field to check) falls through to the max-P50 wall-clock
+      // computation below; every stageShape rule (including taskStageSkew) gets
+      // its own resourceOnly formula here, each returning early (not `break`,
+      // which would fall off the end of this function's switch and implicitly
+      // return `undefined` instead of `null` since the switch is the function's
+      // final statement).
+      if (finding.type === 'stageShape') {
+        if (finding.rule === 'lowParallelism') {
+          const stage = stages.get(finding.stageId          );
+          if (!stage) return null;
+          const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+          const idleCoreMs =
+            Math.max(0, ((finding.totalCores                      ) ?? 0) - (stage.taskCount ?? 0)) * stageDurationMs;
+          // Real per-stage data (cores, task count, duration), no assumed constant.
+          return costOnly('measured', { value: idleCoreMs, unit: 'coreMs' });
+        }
+        if (finding.rule === 'dataExplosion') {
+          const stage = stages.get(finding.stageId          );
+          if (!stage) return null;
+          const excessBytes = Math.max(0, (stage.outputBytes ?? 0) - (stage.inputBytes ?? 0));
+          // Measured input/output byte counts, no assumed constant.
+          return costOnly('measured', { value: excessBytes, unit: 'bytes' });
+        }
+        if (finding.rule === 'taskStageSkew') {
+          const stage = stages.get(finding.stageId          );
+          if (!stage) return null;
+          const totalCores = (finding.totalCores                      ) ?? 0;
+          const taskCount = stage.taskCount ?? 0;
+          // Cores idle during the straggler's tail, at achieved concurrency (not full
+          // cluster capacity, which is lowParallelism's own territory): satisfying this
+          // rule's trigger condition mathematically forces the occupancy-clipped
+          // wall-clock estimate to zero on every firing, so this is resourceOnly, not a
+          // wall-clock claim (see the Overlap caveat section in
+          // docs-site/contributor-guide/architecture/impact-estimation.md).
+          const idleCoreMs =
+            Math.max(0, Math.min(totalCores, taskCount) - 1) *
+            Math.max(0, (stage.taskDurationMax ?? 0) - (stage.taskDurationP50 ?? 0));
+          return costOnly('measured', { value: idleCoreMs, unit: 'coreMs' });
+        }
+        return null;
+      }
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const wasteMs = Math.max(0, (stage.taskDurationMax ?? 0) - (stage.taskDurationP50 ?? 0));
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+    }
+    case 'slowHost': {
+      // Three duration-based shapes, each carrying its absolute-ms figure under a
+      // different field (`value` is a ratio or a share in all three, never ms):
+      //   - the per-host mean branch, discriminated by `metric` (it sets no `variant`)
+      //   - the duration-share branch, discriminated by `variant`
+      //   - the multi-dimension branch, only for its taskTime dimension
+      // Every other shape (the byte-based multiDim dimensions) has no absolute
+      // figure today, so it stays informational.
+      const absoluteMs =
+        finding.metric === 'hostMeanRatio' || finding.variant === 'durationShare'
+          ? (finding.hostMeanMs                      )
+          : finding.variant === 'multiDim' && finding.dimension === 'taskTime'
+            ? (finding.execMaxValue                      )
+            : null;
+      if (absoluteMs == null) {
+        return costOnly('none'); // byte-based multiDim dims: no absolute figure today, no model applied
+      }
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const wasteMs = Math.max(0, absoluteMs - (stage.taskDurationP50 ?? 0));
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+    }
+    case 'duplicatePlanSubtree': {
+      const stageIds = finding.stageIds                        ;
+      if (!stageIds || stageIds.length === 0) return null;
+      // The detector reports metric: 'subtreeOccurrences', value: <occurrences>, always
+      // >= 2 (its own `minOccurrences` threshold). Only the repeats past the first are
+      // redundant: computing the subtree once is real work, so the waste is
+      // (occurrences - 1) / occurrences of the contributing stages' time, not all of it.
+      const occurrences = typeof finding.value === 'number' ? finding.value : 0;
+      // Defensive only: the detector's own `minOccurrences` threshold guarantees occurrences
+      // >= 2 on real data, so this is a malformed-`value` fallback, not a real formula run.
+      if (occurrences < 2) return costOnly('none');
+      const redundantFraction = (occurrences - 1) / occurrences;
+      const wasteMsByStage = new Map                ();
+      for (const id of stageIds) {
+        const s = stages.get(id);
+        if (s) {
+          const durationMs = Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0));
+          wasteMsByStage.set(id, durationMs * redundantFraction);
+        }
+      }
+      const totalWasteMs = [...wasteMsByStage.values()].reduce((sum, ms) => sum + ms, 0);
+      const rawWaste                 = { value: totalWasteMs, unit: 'ms' };
+      const est = estimateMultiStage(stageIds, wasteMsByStage, stages                                          , occupancy);
+      if (!est) return costOnly('measured', rawWaste);
+      return { basis: est.basis, wallClock: est.wallClock, estimateMethod: 'measured', rawWaste };
+    }
+    case 'shuffle': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const shuffleReadBytes = stage.shuffleReadBytes ?? 0;
+      const wasteMs = (shuffleReadBytes / SHUFFLE_THROUGHPUT_BPS) * 1000;
+      // The measured byte volume driving the modeled ms figure above.
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: shuffleReadBytes, unit: 'bytes' });
+    }
+    case 'spill': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const diskBytesSpilled = stage.diskBytesSpilled ?? 0;
+      const wasteMs = (diskBytesSpilled / SPILL_IO_THROUGHPUT_BPS) * 1000;
+      // Surfaces the number the formula actually uses: the finding's displayed `metric`
+      // is memoryBytesSpilled, but disk spill is what costs I/O time.
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: diskBytesSpilled, unit: 'bytes' });
+    }
+    case 'stageSlowness': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+      const wasteMs = Math.max(0, stageDurationMs - STAGE_SLOWNESS_THRESHOLD_MINUTES * 60000);
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' });
+    }
+    case 'partitionSizing': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      let wasteMs = 0;
+      if (finding.rule === 'maxPartitionTooBig') {
+        wasteMs = ((stage.shuffleReadMax ?? 0) / SHUFFLE_THROUGHPUT_BPS) * 1000;
+      } else if (finding.rule === 'shufflePartitionSkew') {
+        const delta = Math.max(0, (stage.shuffleReadMax ?? 0) - (stage.shuffleReadP50 ?? 0));
+        wasteMs = (delta / SHUFFLE_THROUGHPUT_BPS) * 1000;
+      } else if (finding.rule === 'lowShuffleParallelism') {
+        const targetTaskCount = Math.ceil((stage.shuffleReadBytes ?? 0) / IDEAL_BYTES_PER_PARTITION_TASK);
+        const taskCount = stage.taskCount ?? 0;
+        if (targetTaskCount > taskCount && taskCount > 0) {
+          const stageDurationMs = Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
+          // Too few shuffle partitions means each task processes more bytes than the ideal
+          // target, doing serially what more partitions would let run concurrently: the waste
+          // is that serialized work, not the scheduling cost of the tasks you'd add to fix it
+          // (adding tasks INCURS overhead, it doesn't recover any). Model the achievable
+          // duration at target parallelism by scaling down proportionally to the partition
+          // shortfall, and claim the difference.
+          wasteMs = stageDurationMs * (1 - taskCount / targetTaskCount);
+        }
+      } else {
+        return null;
+      }
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' });
+    }
+    case 'tinyTask': {
+      if (finding.stageId == null) return null;
+      const stage = stages.get(finding.stageId);
+      if (!stage) return null;
+      const taskCount = stage.taskCount ?? 0;
+      const excessTaskCount = Math.max(0, taskCount - Math.round(taskCount / 10));
+      const wasteMs = excessTaskCount * TASK_SCHEDULING_OVERHEAD_MS;
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' });
+    }
+    case 'smallFiles': {
+      const wasteMs = ((finding.fileCount                      ) ?? 0) * FILE_OPEN_OVERHEAD_MS;
+      return stageMappableWasteOrCostOnly(wasteMs, finding.stageIds                        , stages, occupancy);
+    }
+    case 'overBroadcast': {
+      // metric: 'broadcastBytes', value: <bytes>.
+      const wasteMs = (((finding.value                      ) ?? 0) / BROADCAST_BANDWIDTH_BPS) * 1000;
+      return stageMappableWasteOrCostOnly(wasteMs, finding.stageIds                        , stages, occupancy);
+    }
+    case 'underBroadcast': {
+      // metric: 'smallerSideBytes', value: <bytes of the smaller join side>.
+      const wasteMs = (((finding.value                      ) ?? 0) / BROADCAST_BANDWIDTH_BPS) * 1000;
+      return stageMappableWasteOrCostOnly(wasteMs, finding.stageIds                        , stages, occupancy);
+    }
+    case 'memoryUtilization': {
+      // The wasteModel variant reports metric: 'wastedMBSeconds', value: <MB-seconds>.
+      if (finding.variant === 'wasteModel' && typeof finding.value === 'number') {
+        return costOnly('measured', { value: finding.value, unit: 'mbSeconds' });
+      }
+      if (finding.variant === 'idleCores') {
+        // Idle core-time priced as memory held but unused: the same MB-seconds unit
+        // the wasteModel variant reports, so the two are comparable.
+        const idleRateFraction = finding.idleRateFraction                      ;
+        const allocatedMB = finding.allocatedMB                      ;
+        const peakExecutors = finding.peakExecutors                      ;
+        const appDurationMs = finding.appDurationMs                      ;
+        if (idleRateFraction != null && allocatedMB != null && peakExecutors != null && appDurationMs != null) {
+          const wastedMBSeconds = idleRateFraction * allocatedMB * peakExecutors * (appDurationMs / 1000);
+          return costOnly('modeled', { value: wastedMBSeconds, unit: 'mbSeconds' });
+        }
+        return costOnly('modeled');
+      }
+      // Only the over-provisioned band is a waste; the near-capacity band is an OOM-risk
+      // signal with no magnitude to report, and the dataUnavailable shape has no inputs
+      // at all: both stay informational.
+      if (finding.variant === 'memoryBand' && finding.rule === 'heapOverProvisioned') {
+        const allocatedBytes = finding.allocatedBytes                      ;
+        const heap = finding.heap                      ;
+        const appDurationMs = finding.appDurationMs                      ;
+        if (allocatedBytes != null && heap != null && appDurationMs != null) {
+          const unusedMB = (allocatedBytes - heap) / (1024 * 1024);
+          const wastedMBSeconds = unusedMB * (appDurationMs / 1000);
+          return costOnly('modeled', { value: wastedMBSeconds, unit: 'mbSeconds' });
+        }
+      }
+      return costOnly('modeled');
+    }
+    case 'utilization': {
+      const fraction = finding.utilizationFraction                      ;
+      const appDurationMs = finding.appDurationMs                      ;
+      const totalCores = finding.totalCores                      ;
+      if (fraction == null || appDurationMs == null || totalCores == null) {
+        return costOnly('measured');
+      }
+      const idleCoreHours = (1 - fraction) * appDurationMs * totalCores / 3.6e6;
+      return costOnly('measured', { value: idleCoreHours, unit: 'coreHours' });
+    }
+    case 'coreLocality': {
+      const nonLocal = (finding.nonLocalTaskCount                      ) ?? 0;
+      const coreMs = nonLocal * NETWORK_FETCH_PENALTY_MS;
+      return costOnly('modeled', { value: coreMs, unit: 'coreMs' });
+    }
+    case 'autoscalingChurn': {
+      const shortLived = (finding.shortLivedExecutorCount                      ) ?? 0;
+      const executorHours = (shortLived * EXECUTOR_STARTUP_OVERHEAD_MS) / 3.6e6;
+      return costOnly('modeled', { value: executorHours, unit: 'coreHours' });
+    }
+    case 'configAudit': {
+      return costOnly('none'); // purely informational: no waste model applied
+    }
+    case 'jobFailureRate': {
+      const failedJobs = (finding.failedJobs                      ) ?? 0;
+      const avgJobDurationMs = (finding.avgJobDurationMs                      ) ?? 0;
+      const coreHoursIsh = (failedJobs * avgJobDurationMs) / 3.6e6;
+      return costOnly('modeled', { value: coreHoursIsh, unit: 'coreHours' });
+    }
+    case 'cachingOpportunity': {
+      const totalReadBytes = (finding.totalReadBytes                      ) ?? 0;
+      const wasteMs = (totalReadBytes / RE_READ_THROUGHPUT_BPS) * 1000;
+      return costOnly('modeled', { value: wasteMs, unit: 'ms' });
+    }
+    case 'cacheUtilization': {
+      const memorySize = (finding.memorySize                      ) ?? 0;
+      const diskSize = (finding.diskSize                      ) ?? 0;
+      const numCachedPartitions = (finding.numCachedPartitions                      ) ?? 0;
+      const numPartitions = (finding.numPartitions                      ) ?? 0;
+      const numUncachedPartitions = Math.max(0, numPartitions - numCachedPartitions);
+      const cachedBytes = memorySize + diskSize;
+      // Extrapolate the never-cached partitions' size from the CACHED partitions' own
+      // average size (uncached/cached, not uncached/total: `numCachedPartitions` partitions
+      // produced `cachedBytes`, not all `numPartitions` of them). `diskSize` is added once
+      // more on its own: those bytes are already cached, but on disk rather than memory, so
+      // re-reading them still costs I/O the way a genuinely-uncached partition would.
+      const uncachedBytes = numCachedPartitions > 0 ? (cachedBytes / numCachedPartitions) * numUncachedPartitions : 0;
+      const uncachedOrSpilledBytes = uncachedBytes + diskSize;
+      const wasteMs = (uncachedOrSpilledBytes / RE_READ_THROUGHPUT_BPS) * 1000;
+      return costOnly('modeled', { value: wasteMs, unit: 'ms' });
+    }
+    case 'stageFailed':
+    case 'failures':
+    case 'incompleteRun': {
+      return costOnly('none'); // purely informational: no waste model applied
+    }
+    default:
+      return null;
+  }
+}
+
+export function estimateImpact(findings           , stages                    , totalCores = 0)            {
+  const occupancy = computeOccupancy(stages                                          , totalCores);
+  for (const f of findings) {
+    const estimate = computeEstimateForFinding(f, stages, occupancy);
+    if (estimate) f.impactEstimate = estimate;
+  }
+  return findings;
+}