sparkforensics-mcp 0.2.0 → 0.2.3

package/vendor-core/impact-estimator.js

@@ -1,14 +1,75 @@
                                                                                                        
-import { computeOccupancy, estimateSingleStage, estimateMultiStage,                                              } from './occupancy.js';
-import { detectorCatalog } from './detectors.js';
+import {
+  computeOccupancy, estimateSingleStage, estimateMultiStage, tailRecoveryMs, tailRemovedWorkMs, stragglerFixLongestTaskMs,
+                                                                                
+} from './occupancy.js';
 
-// Assumed shuffle-network throughput, ~1 Gbps. Starting assumption, unvalidated.
+// Assumed shuffle-network throughput per executor link, ~1 Gbps. Starting assumption, unvalidated.
 const SHUFFLE_THROUGHPUT_BPS = 125_000_000;
-// Assumed disk I/O throughput for spilled data, ~200 MB/s (conservative HDD/SSD blend).
+// Assumed disk I/O throughput per executor for spilled data, ~200 MB/s (conservative HDD/SSD blend).
 const SPILL_IO_THROUGHPUT_BPS = 200_000_000;
+
+// A stage's own per-task overhead: task wall time (launch to finish, summed per executor by
+// finalizeStage) minus executorRunTime, i.e. deserialization, result serialization and the
+// launch round trip that coalescing tasks removes. `concurrency` is the stage's achieved task
+// concurrency (task time / stage duration). Null when the stage has no such data or no overhead.
+function measuredTaskOverhead(stage       )                                                    {
+  const executorStats = Array.isArray(stage.executorStats)
+    ? (stage.executorStats                                ) : [];
+  const taskTimeMs = executorStats.reduce((sum, e) => sum + (e.totalDuration ?? 0), 0);
+  const taskCount = stage.taskCount ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  const overheadMs = taskTimeMs - (stage.executorRunTime ?? 0);
+  if (taskTimeMs <= 0 || taskCount <= 0 || durationMs <= 0 || overheadMs <= 0) return null;
+  return { perTaskMs: overheadMs / taskCount, concurrency: taskTimeMs / durationMs };
+}
+
+// Both constants above are single-device figures (one NIC, one local disk). A stage's shuffle
+// reads and spills are spread over every executor that ran its tasks, each moving its own share
+// in parallel, so the stage's aggregate bandwidth scales with that executor count. Dividing a
+// stage-wide byte total by one device's bandwidth modeled the whole cluster as one link: on 14
+// real logs that claimed up to 1,870s of shuffle time on stages whose tasks measured ~0s of
+// shuffle fetch wait (222 of 284 shuffle findings). No executor data falls back to one device.
+function stageIoParallelism(stage       )         {
+  const executors = Array.isArray(stage.executorStats) ? stage.executorStats.length : 0;
+  return Math.max(1, executors);
+}
+
+// Wall-clock the stage's tasks spent blocked fetching shuffle blocks: fetchWaitTime is a
+// cross-task sum like executorRunTime, so dividing by the stage's average concurrency converts it
+// (the gc estimate's conversion). Null when the stage has no run time or duration to convert with.
+// On 14 real logs the link model claimed 2780s over 284 shuffle findings, 256s once capped at
+// this, with about zero fetch wait on 5 of the 10 non-info ones: reads that overlapped compute
+// stalled nothing.
+function fetchWaitWallClockMs(stage       )                {
+  const fetchWaitMs = stage.fetchWaitTime;
+  const runTimeMs = stage.executorRunTime ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  if (typeof fetchWaitMs !== 'number' || runTimeMs <= 0 || durationMs <= 0) return null;
+  return fetchWaitMs / (runTimeMs / durationMs);
+}
+
+// Wall-clock a stage's wasted (retried) attempts cost it. Each one delayed only its own task, and
+// attempts of different tasks ran side by side: one lost executor fails every task it was running
+// at once (4 wasted attempts of 36.6s each, all first attempts, on a real stage that ran 41 tasks
+// at once, claimed as 146.6s). So the stage lost at most its longest retry chain, the most attempts
+// one task wasted (its highest attempt number + 1) times the mean wasted attempt, or their summed
+// time spread over its slots, whichever is larger. Without a sample of every wasted attempt
+// (retryTaskSamples is capped), the chain isn't known: the summed time.
+function retryWallClockMs(stage       )         {
+  const totalMs = (stage.retryWasteMs                      ) ?? 0;
+  const attempts = (stage.wastedAttempts                      ) ?? 0;
+  const samples = Array.isArray(stage.retryTaskSamples)
+    ? (stage.retryTaskSamples                                ) : [];
+  if (totalMs <= 0 || attempts <= 0 || samples.length < attempts) return totalMs;
+  const chain = samples.reduce((longest, s) => Math.max(longest, (s.attemptNumber ?? 0) + 1), 1);
+  const slots = Math.max(1, stage.peakConcurrentTasks ?? 1);
+  return Math.min(totalMs, Math.max((chain * totalMs) / attempts, totalMs / slots));
+}
 // Spark's classic recommended shuffle partition size.
 const IDEAL_BYTES_PER_PARTITION_TASK = 128 * 1024 * 1024;
-// Assumed per-task scheduling/launch overhead.
+// Assumed per-task scheduling/launch overhead: the fallback when a stage lacks the per-executor
+// task-time sums measuredTaskOverhead needs.
 const TASK_SCHEDULING_OVERHEAD_MS = 50;
 // Assumed per-file open latency (small-file overhead).
 const FILE_OPEN_OVERHEAD_MS = 10;
@@ -21,15 +82,10 @@ const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
 // Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
 const RE_READ_THROUGHPUT_BPS = 125_000_000;
 
-// stageSlowness flags a stage at `infoMin` minutes; that's the floor for the waste this estimate
-// reports. Read from the detector's catalog entry so the two stay in sync automatically.
-const STAGE_SLOWNESS_THRESHOLD_MINUTES = (() => {
-  const infoMin = detectorCatalog().find((d) => d.type === 'stageSlowness')?.thresholds?.infoMin;
-  if (typeof infoMin !== 'number') {
-    throw new Error("impact-estimator: stageSlowness detector's 'infoMin' threshold not found in detectorCatalog()");
-  }
-  return infoMin;
-})();
+// skew and straggler claim time off the stage's longest task itself, so the occupancy clip must
+// not floor them at that same task (see estimateSingleStage). detectors.ts's clippedWasteMs gates
+// both detectors on the same option so the firing floor and the displayed estimate agree.
+const TAIL_CLAIM                             = { shortensLongestTask: true };
 
 // No quantifiable magnitude -> 'informational'; a rawWaste figure with no stage window ->
 // 'resourceOnly'. Never a fake {low:0, high:0}: wallClock is null in both cases.
@@ -46,8 +102,9 @@ function singleStageImpact(
   occupancy                                 ,
   estimateMethod                      ,
   rawWaste                 ,
+  opts                             ,
 )                 {
-  const est = estimateSingleStage(wasteMs, stageId, stages                                          , occupancy);
+  const est = estimateSingleStage(wasteMs, stageId, stages                                          , occupancy, opts);
   if (est) return { basis: est.basis, wallClock: est.wallClock, estimateMethod, rawWaste };
   return costOnly(estimateMethod, rawWaste); // stage excluded from the sweep (duration <= 0)
 }
@@ -77,6 +134,7 @@ function computeEstimateForFinding(
   finding         ,
   stages                    ,
   occupancy                                 ,
+  totalCores        ,
 )                        {
   switch (finding.type) {
     case 'retryWaste': {
@@ -85,7 +143,9 @@ function computeEstimateForFinding(
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
       const wasteMs = (stage.retryWasteMs                      ) ?? 0;
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      const wallClockMs = retryWallClockMs(stage);
+      return singleStageImpact(wallClockMs, finding.stageId, stages, occupancy,
+        wallClockMs === wasteMs ? 'measured' : 'modeled', { value: wasteMs, unit: 'ms' });
     }
     case 'speculationWaste': {
       if (finding.stageId == null) return null;
@@ -103,6 +163,9 @@ function computeEstimateForFinding(
       return { basis: 'serial', wallClock: { low: wasteMs, high: wasteMs }, estimateMethod: 'measured' };
     }
     case 'gc': {
+      // The low-GC branch is an over-provisioning signal whose fix (less executor memory) raises
+      // GC rather than recovering it: the stage's GC time is no saving there, so no waste model.
+      if (finding.direction === 'low') return costOnly('none');
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
@@ -128,8 +191,11 @@ function computeEstimateForFinding(
       const p50 = stage.taskDurationP50 ?? 0;
       // computeSkewRatio's own metric labels (src/detectors.ts): 'P95/median' or 'max/median'.
       const usesP95Branch = finding.metric === 'P95/median';
-      const wasteMs = Math.max(0, usesP95Branch ? (stage.taskDurationP95 ?? 0) - p50 : (stage.taskDurationMax ?? 0) - p50);
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      const singleDelta = Math.max(0, usesP95Branch ? (stage.taskDurationP95 ?? 0) - p50 : (stage.taskDurationMax ?? 0) - p50);
+      const wasteMs = tailRecoveryMs(stage, singleDelta);
+      // Fixing the skew still waits on the longest task it leaves, as for straggler.
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' },
+        { ...TAIL_CLAIM, removedCoreWorkMs: tailRemovedWorkMs(stage, singleDelta), longestTaskAfterFixMs: stragglerFixLongestTaskMs(stage) });
     }
     case 'straggler':
     case 'stageShape': {
@@ -171,8 +237,11 @@ function computeEstimateForFinding(
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
-      const wasteMs = Math.max(0, (stage.taskDurationMax ?? 0) - (stage.taskDurationP50 ?? 0));
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      const longestTaskAfterFixMs = stragglerFixLongestTaskMs(stage);
+      const singleDelta = Math.max(0, (stage.taskDurationMax ?? 0) - longestTaskAfterFixMs);
+      const wasteMs = tailRecoveryMs(stage, singleDelta);
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' },
+        { ...TAIL_CLAIM, removedCoreWorkMs: tailRemovedWorkMs(stage, singleDelta), longestTaskAfterFixMs });
     }
     case 'slowHost': {
       // Three duration-based shapes, each carrying its absolute-ms figure under a different field
@@ -202,18 +271,31 @@ function computeEstimateForFinding(
       const occurrences = typeof finding.value === 'number' ? finding.value : 0;
       // Defensive only: minOccurrences guarantees occurrences >= 2 on real data; a malformed-value fallback.
       if (occurrences < 2) return costOnly('none');
+      // Same-shaped repeats whose details differ compute different data: nothing is known to be
+      // recomputed, so there is no time to claim.
+      if (finding.occurrencesIdentical === false) return costOnly('none');
+      // Each stage contributes the share of its operators inside the repeated subtree: a stage it
+      // shares with other operators (the consuming join, the join's other side) isn't all its
+      // time, and claiming whole stages let sibling groups claim the same stage twice. Findings
+      // built without the field (hand-made fixtures) count every linked stage whole.
+      const shares = (finding.stageShares ?? null)                                 ;
       const redundantFraction = (occurrences - 1) / occurrences;
       const wasteMsByStage = new Map                ();
       for (const id of stageIds) {
         const s = stages.get(id);
-        if (s) {
-          const durationMs = Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0));
-          wasteMsByStage.set(id, durationMs * redundantFraction);
+        const share = shares ? (shares[id] ?? 0) : 1;
+        if (s && share > 0) {
+          // Time with tasks running, not submit-to-complete: a stage left waiting for cores
+          // (2491s open, 60s of tasks on a real log) isn't recomputing anything while it waits.
+          const activeMs = s.taskActiveMs ?? Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0));
+          wasteMsByStage.set(id, activeMs * share * redundantFraction);
         }
       }
+      // No operator of the subtree ran in a known stage: no time to attribute.
+      if (wasteMsByStage.size === 0) return costOnly('none');
       const totalWasteMs = [...wasteMsByStage.values()].reduce((sum, ms) => sum + ms, 0);
       const rawWaste                 = { value: totalWasteMs, unit: 'ms' };
-      const est = estimateMultiStage(stageIds, wasteMsByStage, stages                                          , occupancy);
+      const est = estimateMultiStage([...wasteMsByStage.keys()], wasteMsByStage, stages                                          , occupancy);
       if (!est) return costOnly('measured', rawWaste);
       return { basis: est.basis, wallClock: est.wallClock, estimateMethod: 'measured', rawWaste };
     }
@@ -222,16 +304,21 @@ function computeEstimateForFinding(
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
       const shuffleReadBytes = stage.shuffleReadBytes ?? 0;
-      const wasteMs = (shuffleReadBytes / SHUFFLE_THROUGHPUT_BPS) * 1000;
-      // The measured byte volume driving the modeled ms figure above.
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: shuffleReadBytes, unit: 'bytes' });
+      const modeledMs = (shuffleReadBytes / (SHUFFLE_THROUGHPUT_BPS * stageIoParallelism(stage))) * 1000;
+      // The link model can't see whether the reads stalled the tasks: capped at the fetch wait the
+      // tasks measured, the claim never exceeds what the stage spent blocked on the network.
+      const measuredMs = fetchWaitWallClockMs(stage);
+      const wasteMs = measuredMs == null ? modeledMs : Math.min(modeledMs, measuredMs);
+      // rawWaste: the measured byte volume behind the modeled figure.
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy,
+        measuredMs != null && measuredMs < modeledMs ? 'measured' : 'modeled', { value: shuffleReadBytes, unit: 'bytes' });
     }
     case 'spill': {
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
       const diskBytesSpilled = stage.diskBytesSpilled ?? 0;
-      const wasteMs = (diskBytesSpilled / SPILL_IO_THROUGHPUT_BPS) * 1000;
+      const wasteMs = (diskBytesSpilled / (SPILL_IO_THROUGHPUT_BPS * stageIoParallelism(stage))) * 1000;
       // Surfaces the number the formula uses: the displayed metric is memoryBytesSpilled, but disk spill costs the I/O time.
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: diskBytesSpilled, unit: 'bytes' });
     }
@@ -239,9 +326,21 @@ function computeEstimateForFinding(
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
-      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
-      const wasteMs = Math.max(0, stageDurationMs - STAGE_SLOWNESS_THRESHOLD_MINUTES * 60000);
-      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' });
+      // The recommended fix is more partitions, which only helps a stage that ran fewer tasks than
+      // the cluster has cores: the time its tasks were running could then spread over up to
+      // totalCores (lowShuffleParallelism's shape). Time the stage sat open with no task running
+      // is queueing no partition count recovers. Splitting partitions splits the longest task
+      // too, hence TAIL_CLAIM's post-fix floor. Unknown cluster size: no defensible figure.
+      if (totalCores <= 0) return costOnly('modeled');
+      // A stage that read no input and no shuffle has no data for more partitions to split (a
+      // 1-task count stage open 27 minutes on 5s of CPU was claimed 99% recoverable): claim 0.
+      const readBytes = (stage.inputBytes ?? 0) + (stage.shuffleReadBytes ?? 0);
+      const activeMs = typeof stage.taskActiveMs === 'number'
+        ? stage.taskActiveMs
+        : Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
+      const taskCount = stage.taskCount ?? 0;
+      const wasteMs = readBytes > 0 ? activeMs * Math.max(0, 1 - taskCount / totalCores) : 0;
+      return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' }, TAIL_CLAIM);
     }
     case 'partitionSizing': {
       if (finding.stageId == null) return null;
@@ -275,12 +374,29 @@ function computeEstimateForFinding(
       if (!stage) return null;
       const taskCount = stage.taskCount ?? 0;
       const excessTaskCount = Math.max(0, taskCount - Math.round(taskCount / 10));
+      const measured = measuredTaskOverhead(stage);
+      if (measured) {
+        // Coalescing to a tenth of the tasks removes the excess tasks' per-task overhead: core
+        // time spent in parallel, so wall-clock at the stage's achieved concurrency (floored at
+        // 1: a mostly-idle stage can't save more wall-clock than the task time it removes).
+        const wasteMs = (excessTaskCount * measured.perTaskMs) / Math.max(1, measured.concurrency);
+        return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
+      }
       const wasteMs = excessTaskCount * TASK_SCHEDULING_OVERHEAD_MS;
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' });
     }
     case 'smallFiles': {
-      const wasteMs = ((finding.fileCount                      ) ?? 0) * FILE_OPEN_OVERHEAD_MS;
-      return stageMappableWasteOrCostOnly(wasteMs, finding.stageIds                        , stages, occupancy);
+      const fileMs = ((finding.fileCount                      ) ?? 0) * FILE_OPEN_OVERHEAD_MS;
+      // A read's files are opened by the scan's tasks, in parallel: spread the per-file cost over
+      // the most tasks its stages ran at once (91344 files x 10ms is 913s, claimed against a 117s
+      // stage that ran 314 tasks at once). A write keeps the serial sum: the job commit moves each
+      // output file on the driver, one after another.
+      const stageIds = finding.stageIds                        ;
+      let slots = 1;
+      if (finding.direction === 'read') {
+        for (const id of stageIds ?? []) slots = Math.max(slots, stages.get(id)?.peakConcurrentTasks ?? 1);
+      }
+      return stageMappableWasteOrCostOnly(fileMs / slots, stageIds, stages, occupancy);
     }
     case 'overBroadcast': {
       // metric: 'broadcastBytes', value: <bytes>.
@@ -385,7 +501,7 @@ function computeEstimateForFinding(
 export function estimateImpact(findings           , stages                    , totalCores = 0)            {
   const occupancy = computeOccupancy(stages                                          , totalCores);
   for (const f of findings) {
-    const estimate = computeEstimateForFinding(f, stages, occupancy);
+    const estimate = computeEstimateForFinding(f, stages, occupancy, totalCores);
     if (estimate) f.impactEstimate = estimate;
   }
   return findings;

package/vendor-core/mcp-tools.js

@@ -228,6 +228,11 @@ export function getFindingDocumentation(type        )                       {
   if (tuningPath && anchor && existsSync(tuningPath)) {
     const tuningContent = readFileSync(tuningPath, 'utf8');
     tuningDoc = { anchor, title: extractDocTitle(tuningContent), content: tuningContent };
+  } else if (anchor && !slug) {
+    // A section hosted on a chapter page (e.g. autoscaling churn on cluster-config): return the
+    // owning chapter, the same page the web view's docs link opens.
+    const entry = findNavEntry(pageForAnchor(anchor.replace(/^#/, '')));
+    if (entry) tuningDoc = { anchor, title: entry.title, content: readNavEntryContent(entry) };
   }
 
   return { type, name: titleCase(label), detectionDoc, tuningDoc };
@@ -235,21 +240,29 @@ export function getFindingDocumentation(type        )                       {
 
 const CHAPTERS_NAV_FILE = join(DOCS_CONTENT_DIR, 'chapters', 'nav-index.json');
 
+                                                                                  
+
+function findNavEntry(page        )                       {
+  const nav = JSON.parse(readFileSync(CHAPTERS_NAV_FILE, 'utf8'))              ;
+  return nav.find((e) => e.anchor === page);
+}
+
+function readNavEntryContent(entry          )         {
+  const dir = entry.store === 'tuning' ? 'tuning' : 'chapters';
+  return readFileSync(join(DOCS_CONTENT_DIR, dir, `${entry.slug}.md`), 'utf8');
+}
+
                                                                                  
 
 /** Full tuning-reference markdown for one doc anchor, run-independent. Resolves the anchor to its
  * owning page via pageForAnchor (so '#metric-task-duration' returns the 'metrics' page), looks it up
- * in the committed nav-index, and reads the markdown from the same docs-content store the website
+ * in the generated nav-index, and reads the markdown from the same docs-content store the website
  * renders from. The general-chapter counterpart to getFindingDocumentation (keyed by finding type). */
 export function getReferenceDoc(anchor        )               {
   const page = pageForAnchor(String(anchor).replace(/^#/, ''));
-  const nav = JSON.parse(readFileSync(CHAPTERS_NAV_FILE, 'utf8'))   
-                                                                         ;
-  const entry = nav.find((e) => e.anchor === page);
+  const entry = findNavEntry(page);
   if (!entry) throw mcpError('invalid-anchor', `Unknown reference anchor: ${anchor}`);
-  const dir = entry.store === 'tuning' ? 'tuning' : 'chapters';
-  const content = readFileSync(join(DOCS_CONTENT_DIR, dir, `${entry.slug}.md`), 'utf8');
-  return { anchor: page, title: entry.title, content };
+  return { anchor: page, title: entry.title, content: readNavEntryContent(entry) };
 }
 
 export function getFindingEvidence(

package/vendor-core/occupancy.js

@@ -117,6 +117,9 @@ export function clipToCeiling(wasteMsClaimed        , stage                , cei
                                      
                
                   
+                                                                                                  
+                                                                          
+                        
  
 
 export function computeOccupancy(
@@ -128,7 +131,11 @@ export function computeOccupancy(
   for (const s of stages.values()) {
     const g = gate.get(s.id);
     if (g === undefined) continue; // excluded from the sweep (duration <= 0)
-    info.set(s.id, { gate: g, ceiling: computeCeiling(s, totalCores) });
+    info.set(s.id, {
+      gate: g,
+      ceiling: computeCeiling(s, totalCores),
+      coreWorkFloor: totalCores > 0 ? (s.executorRunTime ?? 0) / totalCores : 0,
+    });
   }
   return info;
 }
@@ -140,6 +147,59 @@ const SERIAL_GATE_THRESHOLD = 0.999;
                                            
  
 
+                            
+                             
+                               
+                          
+                                 
+                           
+                           
+ 
+
+// Wall-clock a skew/straggler fix recovers, given the slowest task's excess over the median. A
+// lone straggler costs that excess; a tail of many slow tasks (a bimodal stage: 26% of 1400
+// tasks over 4x P50 on a real log) costs its summed excess (stragglerExcessMs) spread over the
+// slots the stage had (peakConcurrentTasks), far more than one task's. The task-level replay in
+// dev/eval-tail-replay.mjs recovers about the larger of the two. Average concurrency would be the
+// wrong divisor: a tail-dominated stage runs few tasks for most of its span (5.7 average vs 14
+// peak on one real stage), which doubled the claim.
+export function tailRecoveryMs(stage           , singleTaskExcessMs        )         {
+  const excessMs = stage.stragglerExcessMs ?? 0;
+  const slots = stage.peakConcurrentTasks ?? 0;
+  if (excessMs <= 0 || slots <= 0) return singleTaskExcessMs;
+  return Math.max(singleTaskExcessMs, excessMs / slots);
+}
+
+// Task time a skew/straggler fix removes, by the same measure: the tasks over 4x P50 capped at the
+// median (stragglerExcessMs), or the slowest task's excess when that alone is larger.
+export function tailRemovedWorkMs(stage           , singleTaskExcessMs        )         {
+  return Math.max(singleTaskExcessMs, stage.stragglerExcessMs ?? 0);
+}
+
+// Longest task a straggler fix leaves: every task over 4x P50 comes down to the median, so the
+// longest one at or under that (finalizeStage's longestNonStragglerMs). Without stragglers (a
+// speculation-driven finding) or the field (older snapshots), the median.
+export function stragglerFixLongestTaskMs(stage           )         {
+  const p50 = stage.taskDurationP50 ?? 0;
+  if (!((stage.stragglerCount ?? 0) > 0) || stage.longestNonStragglerMs == null) return p50;
+  return Math.max(p50, stage.longestNonStragglerMs);
+}
+
+                                             
+                                                                                      
+                                                                                              
+                                                                                               
+                                                                                          
+                                
+                                                                                                  
+                                                                                                  
+                                                                                  
+                             
+                                                                                                
+                                                                                  
+                                 
+ 
+
 /**
  * Per-finding estimate for a single-stage waste claim. Returns null when the
  * stage was excluded from the sweep (duration <= 0): callers must fall back
@@ -151,11 +211,19 @@ export function estimateSingleStage(
   stageId        ,
   stages                             ,
   info                                 ,
+  { shortensLongestTask = false, removedCoreWorkMs = 0, longestTaskAfterFixMs = 0 }                             = {},
 )                           {
   const stage = stages.get(stageId);
   const stageInfo = info.get(stageId);
   if (!stage || !stageInfo) return null;
-  const clipped = clipToCeiling(wasteMsClaimed, stage, stageInfo.ceiling);
+  const coreWork = stage.executorRunTime ?? 0;
+  const coreWorkFloor = removedCoreWorkMs > 0 && coreWork > 0
+    ? stageInfo.coreWorkFloor * Math.max(0, 1 - removedCoreWorkMs / coreWork)
+    : stageInfo.coreWorkFloor;
+  const ceiling = shortensLongestTask
+    ? Math.max((stage.taskDurationMax ?? 0) - wasteMsClaimed, longestTaskAfterFixMs, coreWorkFloor)
+    : stageInfo.ceiling;
+  const clipped = clipToCeiling(wasteMsClaimed, stage, ceiling);
   if (stageInfo.gate >= SERIAL_GATE_THRESHOLD) {
     return { basis: 'serial', wallClock: { low: clipped, high: clipped } };
   }

package/vendor-core/parser-worker.js

@@ -2,7 +2,7 @@ import { Gunzip } from './vendor/fflate.js';
 import { createLz4BlockDecoder } from './lz4-block.js';
 import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
 import { createSnappyBlockDecoder } from './snappy-block.js';
-import { createState, dispatchLine, buildChunkDecoder, emitParseCompletion,                  } from './event-handlers.js';
+import { createState, dispatchLine, buildChunkDecoder, emitParseCompletion,                                   } from './event-handlers.js';
 import { TASK_FIELD_NAMES } from './stage-quantiles.js';
 import { runParseFromUrl, sniffCodec } from './shs-fetch.js';
 
@@ -34,7 +34,9 @@ const MIN_PROGRESS_STEPS = 100;
 const PROGRESS_EMIT_LINES = 300;
 
                                      
-                                                     
+// zstdDecoder replaces the vendored fzstd for zstd input; the Node CLI/MCP path passes
+// cli/native-zstd.ts's native-zlib decoder, which a browser bundle can't import.
+                                                                                       
 
 // Minimal shape streamFile/runParse/runParseFiles read off `file` (name, size,
 // slice(start,end).arrayBuffer()): narrower than the full DOM `File`. A real
@@ -59,6 +61,10 @@ const PROGRESS_EMIT_LINES = 300;
 // without touching the vendored files (mirrors shs-fetch.ts's shim).
                                                                            
                                                                                            
+// A Node decoder may decompress off the main thread: streamFile awaits each push.
+                                                                            
+                                                                 
+  
 
 // Stream one File's (possibly compressed) bytes through the codec dispatch,
 // in `chunkSize` slices, invoking `onChunk` with each decompressed buffer as
@@ -69,6 +75,7 @@ export async function streamFile(
   file            ,
   onChunk                                           ,
   chunkSize        ,
+  zstdDecoder                     ,
 )                {
   const header = new Uint8Array(await file.slice(0, Math.min(8, file.size)).arrayBuffer());
   const codec = sniffCodec(header);
@@ -81,7 +88,10 @@ export async function streamFile(
   let currentPct = 0;
   const gunzip = codec === 'gz' ? new (Gunzip                                   )((inflated) => onChunk(inflated, currentPct)) : null;
   const lz4 = codec === 'lz4' ? createLz4BlockDecoder((inflated) => onChunk(inflated, currentPct)) : null;
-  const zstd = codec === 'zstd' ? new (ZstdDecompress                                   )((inflated) => onChunk(inflated, currentPct)) : null;
+  const onZstdChunk = (inflated            ) => onChunk(inflated, currentPct);
+  const zstd = codec !== 'zstd' ? null
+    : zstdDecoder ? zstdDecoder(onZstdChunk)
+      : new (ZstdDecompress                                   )(onZstdChunk);
   const snappy = codec === 'snappy' ? createSnappyBlockDecoder((inflated) => onChunk(inflated, currentPct)) : null;
 
   // A fixed read size gives too few progress checkpoints on smaller files
@@ -99,7 +109,7 @@ export async function streamFile(
     currentPct = start / file.size;
     if (gunzip) gunzip.push(slice, final);
     else if (lz4) lz4.push(slice);
-    else if (zstd) zstd.push(slice, final);
+    else if (zstd) await zstd.push(slice, final);
     else if (snappy) snappy.push(slice);
     else onChunk(slice, currentPct);
   }
@@ -115,7 +125,7 @@ export async function streamFile(
 export async function runParse(
   file            ,
   state             ,
-  { emit = (msg         ) => self.postMessage(msg), chunkSize = CHUNK_SIZE }          = {},
+  { emit = (msg         ) => self.postMessage(msg), chunkSize = CHUNK_SIZE, zstdDecoder }          = {},
 )                {
   if (file.size === 0) {
     emit({ type: 'error', message: 'File is empty.' });
@@ -123,10 +133,13 @@ export async function runParse(
   }
 
   const decoder = buildChunkDecoder();
+  const joined               = [];
   let linesProcessed = 0;
   const feed = (bytes            , pct         ) => {
-    for (const line of decoder.decode(bytes)) {
-      dispatchLine(line, state, emit);
+    joined.length = 0;
+    const lines = decoder.decode(bytes, joined);
+    for (let i = 0, j = 0; i < lines.length; i++) {
+      dispatchLine(lines[i], state, emit, joined[j]?.index === i ? joined[j++] : undefined);
       linesProcessed++;
       if (linesProcessed % PROGRESS_EMIT_LINES === 0) {
         emit({ type: 'progress', pct: pct ?? null, linesProcessed });
@@ -135,7 +148,7 @@ export async function runParse(
   };
 
   try {
-    await streamFile(file, feed, chunkSize);
+    await streamFile(file, feed, chunkSize, zstdDecoder);
   } catch (e) {
     const message = e instanceof Error ? e.message : String(e);
     emit({ type: 'error', message: `Could not decompress "${file.name}": ${message}` });
@@ -147,7 +160,7 @@ export async function runParse(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: SparkListenerApplicationStart not found.' });
+    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
     return;
   }
 
@@ -163,7 +176,7 @@ export async function runParse(
 export async function runParseFiles(
   files              ,
   state             ,
-  { emit = (msg         ) => self.postMessage(msg), chunkSize = CHUNK_SIZE }          = {},
+  { emit = (msg         ) => self.postMessage(msg), chunkSize = CHUNK_SIZE, zstdDecoder }          = {},
 )                {
   if (files.length === 0) {
     emit({ type: 'error', message: 'Rolling event-log directory contained no event files.' });
@@ -171,13 +184,16 @@ export async function runParseFiles(
   }
 
   const decoder = buildChunkDecoder();
+  const joined               = [];
   let linesProcessed = 0;
   const totalSize = files.reduce((sum, f) => sum + f.size, 0);
   let bytesBeforeCurrentFile = 0;
   let currentFileSize = 0;
   const feed = (bytes            , pct         ) => {
-    for (const line of decoder.decode(bytes)) {
-      dispatchLine(line, state, emit);
+    joined.length = 0;
+    const lines = decoder.decode(bytes, joined);
+    for (let i = 0, j = 0; i < lines.length; i++) {
+      dispatchLine(lines[i], state, emit, joined[j]?.index === i ? joined[j++] : undefined);
       linesProcessed++;
       if (linesProcessed % PROGRESS_EMIT_LINES === 0) {
         const overallPct = totalSize > 0 ? (bytesBeforeCurrentFile + (pct ?? 0) * currentFileSize) / totalSize : null;
@@ -189,7 +205,7 @@ export async function runParseFiles(
   for (const file of files) {
     currentFileSize = file.size;
     try {
-      await streamFile(file, feed, chunkSize);
+      await streamFile(file, feed, chunkSize, zstdDecoder);
     } catch (e) {
       const message = e instanceof Error ? e.message : String(e);
       emit({ type: 'error', message: `Could not decompress "${file.name}": ${message}` });
@@ -203,7 +219,7 @@ export async function runParseFiles(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: SparkListenerApplicationStart not found.' });
+    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
     return;
   }
 

package/vendor-core/plan-summary.js

@@ -55,6 +55,10 @@ export function scanRelationId(name        , detail        )                {
   //    Delta tables). All catalog tables surface as `spark_catalog.<db>.<table>`.
   let format = null, table = null;
   const nameM = name.match(/^Scan\s+(parquet|orc|csv|json)\s+(spark_catalog\.\S+)$/i);
+  // Fast reject: every branch below that returns a key needs this name match, a FileScan
+  // detail or a JDBCRelation detail. Most plan nodes (Project, Filter, joins) have none, and
+  // their long details otherwise pay every regex below.
+  if (!nameM && !/FileScan/i.test(detail) && !detail.includes('JDBCRelation')) return null;
   if (nameM) { format = nameM[1].toLowerCase(); table = nameM[2]; }
   else {
     const detM = detail.match(/FileScan\s+(parquet|orc|csv|json)\s+(spark_catalog\.[^[\s]+)\[/i);