sparkforensics-mcp 0.2.2 → 0.2.4

package/vendor-core/impact-estimator.js

@@ -1,14 +1,76 @@
                                                                                                        
-import { computeOccupancy, estimateSingleStage, estimateMultiStage,                                              } from './occupancy.js';
-import { detectorCatalog } from './detectors.js';
+import { nsToMs } from './format-utils.js';
+import {
+  computeOccupancy, estimateSingleStage, estimateMultiStage, tailRecoveryMs, tailRemovedWorkMs, stragglerFixLongestTaskMs,
+                                                                                
+} from './occupancy.js';
 
-// Assumed shuffle-network throughput, ~1 Gbps. Starting assumption, unvalidated.
+// Assumed shuffle-network throughput per executor link, ~1 Gbps. Starting assumption, unvalidated.
 const SHUFFLE_THROUGHPUT_BPS = 125_000_000;
-// Assumed disk I/O throughput for spilled data, ~200 MB/s (conservative HDD/SSD blend).
+// Assumed disk I/O throughput per executor for spilled data, ~200 MB/s (conservative HDD/SSD blend).
 const SPILL_IO_THROUGHPUT_BPS = 200_000_000;
+
+// A stage's own per-task overhead: task wall time (launch to finish, summed per executor by
+// finalizeStage) minus executorRunTime, i.e. deserialization, result serialization and the
+// launch round trip that coalescing tasks removes. `concurrency` is the stage's achieved task
+// concurrency (task time / stage duration). Null when the stage has no such data or no overhead.
+function measuredTaskOverhead(stage       )                                                    {
+  const executorStats = Array.isArray(stage.executorStats)
+    ? (stage.executorStats                                ) : [];
+  const taskTimeMs = executorStats.reduce((sum, e) => sum + (e.totalDuration ?? 0), 0);
+  const taskCount = stage.taskCount ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  const overheadMs = taskTimeMs - (stage.executorRunTime ?? 0);
+  if (taskTimeMs <= 0 || taskCount <= 0 || durationMs <= 0 || overheadMs <= 0) return null;
+  return { perTaskMs: overheadMs / taskCount, concurrency: taskTimeMs / durationMs };
+}
+
+// Both constants above are single-device figures (one NIC, one local disk). A stage's shuffle
+// reads and spills are spread over every executor that ran its tasks, each moving its own share
+// in parallel, so the stage's aggregate bandwidth scales with that executor count. Dividing a
+// stage-wide byte total by one device's bandwidth modeled the whole cluster as one link: on 14
+// real logs that claimed up to 1,870s of shuffle time on stages whose tasks measured ~0s of
+// shuffle fetch wait (222 of 284 shuffle findings). No executor data falls back to one device.
+function stageIoParallelism(stage       )         {
+  const executors = Array.isArray(stage.executorStats) ? stage.executorStats.length : 0;
+  return Math.max(1, executors);
+}
+
+// Wall-clock the stage's tasks spent blocked fetching shuffle blocks: fetchWaitTime is a
+// cross-task sum like executorRunTime, so dividing by the stage's average concurrency converts it
+// (the gc estimate's conversion). Null when the stage has no run time or duration to convert with.
+// On 14 real logs the link model claimed 2780s over 284 shuffle findings, 256s once capped at
+// this, with about zero fetch wait on 5 of the 10 non-info ones: reads that overlapped compute
+// stalled nothing.
+function fetchWaitWallClockMs(stage       )                {
+  const fetchWaitMs = stage.fetchWaitTime;
+  const runTimeMs = stage.executorRunTime ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  if (typeof fetchWaitMs !== 'number' || runTimeMs <= 0 || durationMs <= 0) return null;
+  return fetchWaitMs / (runTimeMs / durationMs);
+}
+
+// Wall-clock a stage's wasted (retried) attempts cost it. Each one delayed only its own task, and
+// attempts of different tasks ran side by side: one lost executor fails every task it was running
+// at once (4 wasted attempts of 36.6s each, all first attempts, on a real stage that ran 41 tasks
+// at once, claimed as 146.6s). So the stage lost at most its longest retry chain, the most attempts
+// one task wasted (its highest attempt number + 1) times the mean wasted attempt, or their summed
+// time spread over its slots, whichever is larger. Without a sample of every wasted attempt
+// (retryTaskSamples is capped), the chain isn't known: the summed time.
+function retryWallClockMs(stage       )         {
+  const totalMs = (stage.retryWasteMs                      ) ?? 0;
+  const attempts = (stage.wastedAttempts                      ) ?? 0;
+  const samples = Array.isArray(stage.retryTaskSamples)
+    ? (stage.retryTaskSamples                                ) : [];
+  if (totalMs <= 0 || attempts <= 0 || samples.length < attempts) return totalMs;
+  const chain = samples.reduce((longest, s) => Math.max(longest, (s.attemptNumber ?? 0) + 1), 1);
+  const slots = Math.max(1, stage.peakConcurrentTasks ?? 1);
+  return Math.min(totalMs, Math.max((chain * totalMs) / attempts, totalMs / slots));
+}
 // Spark's classic recommended shuffle partition size.
 const IDEAL_BYTES_PER_PARTITION_TASK = 128 * 1024 * 1024;
-// Assumed per-task scheduling/launch overhead.
+// Assumed per-task scheduling/launch overhead: the fallback when a stage lacks the per-executor
+// task-time sums measuredTaskOverhead needs.
 const TASK_SCHEDULING_OVERHEAD_MS = 50;
 // Assumed per-file open latency (small-file overhead).
 const FILE_OPEN_OVERHEAD_MS = 10;
@@ -21,15 +83,28 @@ const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
 // Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
 const RE_READ_THROUGHPUT_BPS = 125_000_000;
 
-// stageSlowness flags a stage at `infoMin` minutes; that's the floor for the waste this estimate
-// reports. Read from the detector's catalog entry so the two stay in sync automatically.
-const STAGE_SLOWNESS_THRESHOLD_MINUTES = (() => {
-  const infoMin = detectorCatalog().find((d) => d.type === 'stageSlowness')?.thresholds?.infoMin;
-  if (typeof infoMin !== 'number') {
-    throw new Error("impact-estimator: stageSlowness detector's 'infoMin' threshold not found in detectorCatalog()");
-  }
-  return infoMin;
-})();
+// Below this share of executorRunTime spent on CPU, a stage's tasks were idle, waiting on something
+// outside Spark: on 14 real logs (2026-09-23) every non-Python stage under 1% was a JDBC read, a
+// file listing or a Delta log read, while file writes, which more partitions do parallelize,
+// start at 2%.
+const IDLE_CPU_SHARE_MAX = 0.01;
+
+// True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
+// the share can't be trusted: no CPU time recorded (older Spark logs omit the metric), or Python
+// code run through PythonRDD, whose worker-process CPU executorCpuTime (the JVM task thread's)
+// never counts (such stages read 0.1% on the same logs while computing).
+function tasksMostlyIdle(stage       )          {
+  const runMs = stage.executorRunTime ?? 0;
+  const cpuMs = nsToMs(stage.executorCpuTime ?? 0);
+  if (runMs <= 0 || cpuMs <= 0) return false;
+  if (/PythonRDD/.test(stage.name ?? '') || /org\.apache\.spark\.api\.python\./.test(stage.details ?? '')) return false;
+  return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
+}
+
+// skew and straggler claim time off the stage's longest task itself, so the occupancy clip must
+// not floor them at that same task (see estimateSingleStage). detectors.ts's clippedWasteMs gates
+// both detectors on the same option so the firing floor and the displayed estimate agree.
+const TAIL_CLAIM                             = { shortensLongestTask: true };
 
 // No quantifiable magnitude -> 'informational'; a rawWaste figure with no stage window ->
 // 'resourceOnly'. Never a fake {low:0, high:0}: wallClock is null in both cases.
@@ -46,8 +121,9 @@ function singleStageImpact(
   occupancy                                 ,
   estimateMethod                      ,
   rawWaste                 ,
+  opts                             ,
 )                 {
-  const est = estimateSingleStage(wasteMs, stageId, stages                                          , occupancy);
+  const est = estimateSingleStage(wasteMs, stageId, stages                                          , occupancy, opts);
   if (est) return { basis: est.basis, wallClock: est.wallClock, estimateMethod, rawWaste };
   return costOnly(estimateMethod, rawWaste); // stage excluded from the sweep (duration <= 0)
 }
@@ -77,6 +153,7 @@ function computeEstimateForFinding(
   finding         ,
   stages                    ,
   occupancy                                 ,
+  totalCores        ,
 )                        {
   switch (finding.type) {
     case 'retryWaste': {
@@ -85,7 +162,9 @@ function computeEstimateForFinding(
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
       const wasteMs = (stage.retryWasteMs                      ) ?? 0;
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      const wallClockMs = retryWallClockMs(stage);
+      return singleStageImpact(wallClockMs, finding.stageId, stages, occupancy,
+        wallClockMs === wasteMs ? 'measured' : 'modeled', { value: wasteMs, unit: 'ms' });
     }
     case 'speculationWaste': {
       if (finding.stageId == null) return null;
@@ -103,6 +182,9 @@ function computeEstimateForFinding(
       return { basis: 'serial', wallClock: { low: wasteMs, high: wasteMs }, estimateMethod: 'measured' };
     }
     case 'gc': {
+      // The low-GC branch is an over-provisioning signal whose fix (less executor memory) raises
+      // GC rather than recovering it: the stage's GC time is no saving there, so no waste model.
+      if (finding.direction === 'low') return costOnly('none');
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
@@ -128,8 +210,11 @@ function computeEstimateForFinding(
       const p50 = stage.taskDurationP50 ?? 0;
       // computeSkewRatio's own metric labels (src/detectors.ts): 'P95/median' or 'max/median'.
       const usesP95Branch = finding.metric === 'P95/median';
-      const wasteMs = Math.max(0, usesP95Branch ? (stage.taskDurationP95 ?? 0) - p50 : (stage.taskDurationMax ?? 0) - p50);
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      const singleDelta = Math.max(0, usesP95Branch ? (stage.taskDurationP95 ?? 0) - p50 : (stage.taskDurationMax ?? 0) - p50);
+      const wasteMs = tailRecoveryMs(stage, singleDelta);
+      // Fixing the skew still waits on the longest task it leaves, as for straggler.
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' },
+        { ...TAIL_CLAIM, removedCoreWorkMs: tailRemovedWorkMs(stage, singleDelta), longestTaskAfterFixMs: stragglerFixLongestTaskMs(stage) });
     }
     case 'straggler':
     case 'stageShape': {
@@ -171,8 +256,11 @@ function computeEstimateForFinding(
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
-      const wasteMs = Math.max(0, (stage.taskDurationMax ?? 0) - (stage.taskDurationP50 ?? 0));
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      const longestTaskAfterFixMs = stragglerFixLongestTaskMs(stage);
+      const singleDelta = Math.max(0, (stage.taskDurationMax ?? 0) - longestTaskAfterFixMs);
+      const wasteMs = tailRecoveryMs(stage, singleDelta);
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' },
+        { ...TAIL_CLAIM, removedCoreWorkMs: tailRemovedWorkMs(stage, singleDelta), longestTaskAfterFixMs });
     }
     case 'slowHost': {
       // Three duration-based shapes, each carrying its absolute-ms figure under a different field
@@ -202,18 +290,31 @@ function computeEstimateForFinding(
       const occurrences = typeof finding.value === 'number' ? finding.value : 0;
       // Defensive only: minOccurrences guarantees occurrences >= 2 on real data; a malformed-value fallback.
       if (occurrences < 2) return costOnly('none');
+      // Same-shaped repeats whose details differ compute different data: nothing is known to be
+      // recomputed, so there is no time to claim.
+      if (finding.occurrencesIdentical === false) return costOnly('none');
+      // Each stage contributes the share of its operators inside the repeated subtree: a stage it
+      // shares with other operators (the consuming join, the join's other side) isn't all its
+      // time, and claiming whole stages let sibling groups claim the same stage twice. Findings
+      // built without the field (hand-made fixtures) count every linked stage whole.
+      const shares = (finding.stageShares ?? null)                                 ;
       const redundantFraction = (occurrences - 1) / occurrences;
       const wasteMsByStage = new Map                ();
       for (const id of stageIds) {
         const s = stages.get(id);
-        if (s) {
-          const durationMs = Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0));
-          wasteMsByStage.set(id, durationMs * redundantFraction);
+        const share = shares ? (shares[id] ?? 0) : 1;
+        if (s && share > 0) {
+          // Time with tasks running, not submit-to-complete: a stage left waiting for cores
+          // (2491s open, 60s of tasks on a real log) isn't recomputing anything while it waits.
+          const activeMs = s.taskActiveMs ?? Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0));
+          wasteMsByStage.set(id, activeMs * share * redundantFraction);
         }
       }
+      // No operator of the subtree ran in a known stage: no time to attribute.
+      if (wasteMsByStage.size === 0) return costOnly('none');
       const totalWasteMs = [...wasteMsByStage.values()].reduce((sum, ms) => sum + ms, 0);
       const rawWaste                 = { value: totalWasteMs, unit: 'ms' };
-      const est = estimateMultiStage(stageIds, wasteMsByStage, stages                                          , occupancy);
+      const est = estimateMultiStage([...wasteMsByStage.keys()], wasteMsByStage, stages                                          , occupancy);
       if (!est) return costOnly('measured', rawWaste);
       return { basis: est.basis, wallClock: est.wallClock, estimateMethod: 'measured', rawWaste };
     }
@@ -222,16 +323,21 @@ function computeEstimateForFinding(
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
       const shuffleReadBytes = stage.shuffleReadBytes ?? 0;
-      const wasteMs = (shuffleReadBytes / SHUFFLE_THROUGHPUT_BPS) * 1000;
-      // The measured byte volume driving the modeled ms figure above.
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: shuffleReadBytes, unit: 'bytes' });
+      const modeledMs = (shuffleReadBytes / (SHUFFLE_THROUGHPUT_BPS * stageIoParallelism(stage))) * 1000;
+      // The link model can't see whether the reads stalled the tasks: capped at the fetch wait the
+      // tasks measured, the claim never exceeds what the stage spent blocked on the network.
+      const measuredMs = fetchWaitWallClockMs(stage);
+      const wasteMs = measuredMs == null ? modeledMs : Math.min(modeledMs, measuredMs);
+      // rawWaste: the measured byte volume behind the modeled figure.
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy,
+        measuredMs != null && measuredMs < modeledMs ? 'measured' : 'modeled', { value: shuffleReadBytes, unit: 'bytes' });
     }
     case 'spill': {
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
       const diskBytesSpilled = stage.diskBytesSpilled ?? 0;
-      const wasteMs = (diskBytesSpilled / SPILL_IO_THROUGHPUT_BPS) * 1000;
+      const wasteMs = (diskBytesSpilled / (SPILL_IO_THROUGHPUT_BPS * stageIoParallelism(stage))) * 1000;
       // Surfaces the number the formula uses: the displayed metric is memoryBytesSpilled, but disk spill costs the I/O time.
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: diskBytesSpilled, unit: 'bytes' });
     }
@@ -239,9 +345,22 @@ function computeEstimateForFinding(
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
-      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
-      const wasteMs = Math.max(0, stageDurationMs - STAGE_SLOWNESS_THRESHOLD_MINUTES * 60000);
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' });
+      // The recommended fix is more partitions, which only helps a stage that ran fewer tasks than
+      // the cluster has cores: the time its tasks were running could then spread over up to
+      // totalCores (lowShuffleParallelism's shape). Time the stage sat open with no task running
+      // is queueing no partition count recovers. Splitting partitions splits the longest task
+      // too, hence TAIL_CLAIM's post-fix floor. Unknown cluster size: no defensible figure.
+      if (totalCores <= 0) return costOnly('modeled');
+      // A stage that read no input and no shuffle, its tasks idle waiting on an external system,
+      // gains nothing from more partitions (a 1-task JDBC count stage open 27 minutes on 5s of CPU
+      // was claimed 99% recoverable): claim 0. Stages that read bytes keep their claim.
+      const readBytes = (stage.inputBytes ?? 0) + (stage.shuffleReadBytes ?? 0);
+      const activeMs = typeof stage.taskActiveMs === 'number'
+        ? stage.taskActiveMs
+        : Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
+      const taskCount = stage.taskCount ?? 0;
+      const wasteMs = readBytes <= 0 && tasksMostlyIdle(stage) ? 0 : activeMs * Math.max(0, 1 - taskCount / totalCores);
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' }, TAIL_CLAIM);
     }
     case 'partitionSizing': {
       if (finding.stageId == null) return null;
@@ -275,12 +394,29 @@ function computeEstimateForFinding(
       if (!stage) return null;
       const taskCount = stage.taskCount ?? 0;
       const excessTaskCount = Math.max(0, taskCount - Math.round(taskCount / 10));
+      const measured = measuredTaskOverhead(stage);
+      if (measured) {
+        // Coalescing to a tenth of the tasks removes the excess tasks' per-task overhead: core
+        // time spent in parallel, so wall-clock at the stage's achieved concurrency (floored at
+        // 1: a mostly-idle stage can't save more wall-clock than the task time it removes).
+        const wasteMs = (excessTaskCount * measured.perTaskMs) / Math.max(1, measured.concurrency);
+        return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      }
       const wasteMs = excessTaskCount * TASK_SCHEDULING_OVERHEAD_MS;
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' });
     }
     case 'smallFiles': {
-      const wasteMs = ((finding.fileCount                      ) ?? 0) * FILE_OPEN_OVERHEAD_MS;
-      return stageMappableWasteOrCostOnly(wasteMs, finding.stageIds                        , stages, occupancy);
+      const fileMs = ((finding.fileCount                      ) ?? 0) * FILE_OPEN_OVERHEAD_MS;
+      // A read's files are opened by the scan's tasks, in parallel: spread the per-file cost over
+      // the most tasks its stages ran at once (91344 files x 10ms is 913s, claimed against a 117s
+      // stage that ran 314 tasks at once). A write keeps the serial sum: the job commit moves each
+      // output file on the driver, one after another.
+      const stageIds = finding.stageIds                        ;
+      let slots = 1;
+      if (finding.direction === 'read') {
+        for (const id of stageIds ?? []) slots = Math.max(slots, stages.get(id)?.peakConcurrentTasks ?? 1);
+      }
+      return stageMappableWasteOrCostOnly(fileMs / slots, stageIds, stages, occupancy);
     }
     case 'overBroadcast': {
       // metric: 'broadcastBytes', value: <bytes>.
@@ -385,7 +521,7 @@ function computeEstimateForFinding(
 export function estimateImpact(findings           , stages                    , totalCores = 0)            {
   const occupancy = computeOccupancy(stages                                          , totalCores);
   for (const f of findings) {
-    const estimate = computeEstimateForFinding(f, stages, occupancy);
+    const estimate = computeEstimateForFinding(f, stages, occupancy, totalCores);
     if (estimate) f.impactEstimate = estimate;
   }
   return findings;

package/vendor-core/mcp-tools.js

@@ -228,6 +228,11 @@ export function getFindingDocumentation(type        )                       {
   if (tuningPath && anchor && existsSync(tuningPath)) {
     const tuningContent = readFileSync(tuningPath, 'utf8');
     tuningDoc = { anchor, title: extractDocTitle(tuningContent), content: tuningContent };
+  } else if (anchor && !slug) {
+    // A section hosted on a chapter page (e.g. autoscaling churn on cluster-config): return the
+    // owning chapter, the same page the web view's docs link opens.
+    const entry = findNavEntry(pageForAnchor(anchor.replace(/^#/, '')));
+    if (entry) tuningDoc = { anchor, title: entry.title, content: readNavEntryContent(entry) };
   }
 
   return { type, name: titleCase(label), detectionDoc, tuningDoc };
@@ -235,21 +240,29 @@ export function getFindingDocumentation(type        )                       {
 
 const CHAPTERS_NAV_FILE = join(DOCS_CONTENT_DIR, 'chapters', 'nav-index.json');
 
+                                                                                  
+
+function findNavEntry(page        )                       {
+  const nav = JSON.parse(readFileSync(CHAPTERS_NAV_FILE, 'utf8'))              ;
+  return nav.find((e) => e.anchor === page);
+}
+
+function readNavEntryContent(entry          )         {
+  const dir = entry.store === 'tuning' ? 'tuning' : 'chapters';
+  return readFileSync(join(DOCS_CONTENT_DIR, dir, `${entry.slug}.md`), 'utf8');
+}
+
                                                                                  
 
 /** Full tuning-reference markdown for one doc anchor, run-independent. Resolves the anchor to its
  * owning page via pageForAnchor (so '#metric-task-duration' returns the 'metrics' page), looks it up
- * in the committed nav-index, and reads the markdown from the same docs-content store the website
+ * in the generated nav-index, and reads the markdown from the same docs-content store the website
  * renders from. The general-chapter counterpart to getFindingDocumentation (keyed by finding type). */
 export function getReferenceDoc(anchor        )               {
   const page = pageForAnchor(String(anchor).replace(/^#/, ''));
-  const nav = JSON.parse(readFileSync(CHAPTERS_NAV_FILE, 'utf8'))   
-                                                                         ;
-  const entry = nav.find((e) => e.anchor === page);
+  const entry = findNavEntry(page);
   if (!entry) throw mcpError('invalid-anchor', `Unknown reference anchor: ${anchor}`);
-  const dir = entry.store === 'tuning' ? 'tuning' : 'chapters';
-  const content = readFileSync(join(DOCS_CONTENT_DIR, dir, `${entry.slug}.md`), 'utf8');
-  return { anchor: page, title: entry.title, content };
+  return { anchor: page, title: entry.title, content: readNavEntryContent(entry) };
 }
 
 export function getFindingEvidence(

package/vendor-core/occupancy.js

@@ -117,6 +117,9 @@ export function clipToCeiling(wasteMsClaimed        , stage                , cei
                                      
                
                   
+                                                                                                  
+                                                                          
+                        
  
 
 export function computeOccupancy(
@@ -128,7 +131,11 @@ export function computeOccupancy(
   for (const s of stages.values()) {
     const g = gate.get(s.id);
     if (g === undefined) continue; // excluded from the sweep (duration <= 0)
-    info.set(s.id, { gate: g, ceiling: computeCeiling(s, totalCores) });
+    info.set(s.id, {
+      gate: g,
+      ceiling: computeCeiling(s, totalCores),
+      coreWorkFloor: totalCores > 0 ? (s.executorRunTime ?? 0) / totalCores : 0,
+    });
   }
   return info;
 }
@@ -140,6 +147,60 @@ const SERIAL_GATE_THRESHOLD = 0.999;
                                            
  
 
+                            
+                             
+                               
+                          
+                                 
+                                
+                           
+                           
+ 
+
+// Wall-clock a skew/straggler fix recovers: finalizeStage's task-level replay
+// (tailReplayRecoveryMs, computeTailReplayRecoveryMs) when the stage carries it. Without it (a
+// stage built by hand, as in detector tests) an estimate from the slowest task's excess over the
+// median: a lone straggler costs that excess; a tail of many slow tasks costs its summed excess
+// (stragglerExcessMs) spread over the slots the stage had (peakConcurrentTasks), and the replay
+// recovers about the larger of the two.
+export function tailRecoveryMs(stage           , singleTaskExcessMs        )         {
+  if (stage.tailReplayRecoveryMs != null) return stage.tailReplayRecoveryMs;
+  const excessMs = stage.stragglerExcessMs ?? 0;
+  const slots = stage.peakConcurrentTasks ?? 0;
+  if (excessMs <= 0 || slots <= 0) return singleTaskExcessMs;
+  return Math.max(singleTaskExcessMs, excessMs / slots);
+}
+
+// Task time a skew/straggler fix removes, by the same measure: the tasks over 4x P50 capped at the
+// median (stragglerExcessMs), or the slowest task's excess when that alone is larger.
+export function tailRemovedWorkMs(stage           , singleTaskExcessMs        )         {
+  return Math.max(singleTaskExcessMs, stage.stragglerExcessMs ?? 0);
+}
+
+// Longest task a straggler fix leaves: every task over 4x P50 comes down to the median, so the
+// longest one at or under that (finalizeStage's longestNonStragglerMs). Without stragglers (a
+// speculation-driven finding) or the field (older snapshots), the median.
+export function stragglerFixLongestTaskMs(stage           )         {
+  const p50 = stage.taskDurationP50 ?? 0;
+  if (!((stage.stragglerCount ?? 0) > 0) || stage.longestNonStragglerMs == null) return p50;
+  return Math.max(p50, stage.longestNonStragglerMs);
+}
+
+                                             
+                                                                                      
+                                                                                              
+                                                                                               
+                                                                                          
+                                
+                                                                                                  
+                                                                                                  
+                                                                                  
+                             
+                                                                                                
+                                                                                  
+                                 
+ 
+
 /**
  * Per-finding estimate for a single-stage waste claim. Returns null when the
  * stage was excluded from the sweep (duration <= 0): callers must fall back
@@ -151,11 +212,19 @@ export function estimateSingleStage(
   stageId        ,
   stages                             ,
   info                                 ,
+  { shortensLongestTask = false, removedCoreWorkMs = 0, longestTaskAfterFixMs = 0 }                             = {},
 )                           {
   const stage = stages.get(stageId);
   const stageInfo = info.get(stageId);
   if (!stage || !stageInfo) return null;
-  const clipped = clipToCeiling(wasteMsClaimed, stage, stageInfo.ceiling);
+  const coreWork = stage.executorRunTime ?? 0;
+  const coreWorkFloor = removedCoreWorkMs > 0 && coreWork > 0
+    ? stageInfo.coreWorkFloor * Math.max(0, 1 - removedCoreWorkMs / coreWork)
+    : stageInfo.coreWorkFloor;
+  const ceiling = shortensLongestTask
+    ? Math.max((stage.taskDurationMax ?? 0) - wasteMsClaimed, longestTaskAfterFixMs, coreWorkFloor)
+    : stageInfo.ceiling;
+  const clipped = clipToCeiling(wasteMsClaimed, stage, ceiling);
   if (stageInfo.gate >= SERIAL_GATE_THRESHOLD) {
     return { basis: 'serial', wallClock: { low: clipped, high: clipped } };
   }