sparkforensics-mcp 0.4.0 → 0.5.0

package/vendor-core/finding-presentation.js

@@ -8,6 +8,7 @@
 // detectors.ts is type-only, so it is erased at build time. A detector's scope, order and emits
 // list stay on its DETECTORS entry; renderers get them through detectorInfoByType().
                                                    
+                                                      
                                                          
 
                                                              
@@ -26,6 +27,28 @@
                                                                    
  
 
+/** A share threshold as captions and caveats state it: 0.005 -> "0.5%", never float noise like 7.000000000000001%. */
+export const shareLabel = (share        )         => `${Math.round(share * 1e6) / 1e4}%`;
+
+// Whether the detector found `key` already logged on for this run. Its switchFix (detectors.ts)
+// then worded the row's own text for that case and left the property out of the remediation, so
+// a generic line reads the same decision and never recommends a switch the row says is on.
+// A finding with no remediation (older or hand-built data) keeps the property wording.
+function switchAlreadyOn(finding                                 , key        )          {
+  return finding.remediation != null && !finding.remediation.some((r) => r.key === key);
+}
+
+const SKEW_JOIN_KEY = 'spark.sql.adaptive.skewJoin.enabled';
+const SKEW_JOIN_ALREADY_ON = 'AQE skew-join handling is already on, so salt the key or repartition on a better key.';
+const SKEW_JOIN_AQE_OFF = 'AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise salt the key or repartition on a better key.';
+
+// The skew-join generic line, worded per the row's remediation: AQE logged off, switch already on, or neither.
+function skewJoinGeneric(f                                 , unset        )         {
+  if (f.remediation?.some((r) => r.key === 'spark.sql.adaptive.enabled')) return SKEW_JOIN_AQE_OFF;
+  return switchAlreadyOn(f, SKEW_JOIN_KEY) ? SKEW_JOIN_ALREADY_ON : unset;
+}
+const DYNAMIC_ALLOCATION_KEY = 'spark.dynamicAllocation.enabled';
+
 // The four configAudit DETECTORS entries share this row, one per audited property.
 const CONFIG_AUDIT_PRESENTATION                                     = {
   name: 'config audit',
@@ -58,7 +81,7 @@ export const FINDING_PRESENTATION
   incompleteRun: {
     name: 'incomplete run',
     tag: 'INCMP',
-    thresholdSummary: () => 'an event log missing its terminal ApplicationEnd/job-completion event',
+    thresholdSummary: () => 'an event log missing its terminal ApplicationEnd event',
     actionLabel: () => undefined,
     genericRecommendation: () => undefined,
   },
@@ -66,14 +89,15 @@ export const FINDING_PRESENTATION
   skew: {
     name: 'task skew',
     tag: 'SKEW',
-    thresholdSummary: () => 'task duration skew above the configured ratio',
+    thresholdSummary: (t) => `P95 task time over ${t.ratioWarn}× the median (the longest task on stages under ${t.minTasksForP95} tasks)`,
     actionLabel: () => 'Fix task skew',
-    genericRecommendation: () => 'For join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key to reduce task skew.',
+    genericRecommendation: (f) => skewJoinGeneric(f,
+      'For join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.'),
   },
   stageShape: {
     name: 'stage shape',
     tag: 'SHAPE',
-    thresholdSummary: () => 'low parallelism, data explosion, or task-count skew relative to core count',
+    thresholdSummary: (t) => `under ${t.pRatioMax} tasks per core, output over ${t.oiRatioMax}× input, or one task spanning over ${Math.round(t.stageShareMin * 100)}% of the stage's wall-clock at over ${t.skewWarn}× the median task`,
     actionLabel(f) {
       switch (f.rule) {
         case 'lowParallelism': return 'Increase parallelism';
@@ -94,7 +118,7 @@ export const FINDING_PRESENTATION
   tinyTask: {
     name: 'tiny tasks',
     tag: 'TINY',
-    thresholdSummary: () => 'median task duration below the configured floor',
+    thresholdSummary: (t) => `${t.minTasks}+ tasks with a median of ${t.maxP50}ms or less and a P95 of ${t.maxP95}ms or less`,
     actionLabel: () => 'Coalesce small tasks',
     // The shuffle-vs-no-shuffle fix isn't a Finding field, so one sentence covers both.
     genericRecommendation: () => 'Scheduler overhead may dominate: lower spark.sql.shuffle.partitions, or coalesce down to fewer, larger tasks.',
@@ -103,14 +127,14 @@ export const FINDING_PRESENTATION
   shuffle: {
     name: 'shuffle I/O',
     tag: 'SHFL',
-    thresholdSummary: () => 'shuffle read above the configured minimum byte threshold',
+    thresholdSummary: (t) => `stage shuffle read above ${t.minBytes / 1048576} MiB`,
     actionLabel: () => 'Reduce shuffle size',
-    genericRecommendation: () => 'Consider increasing spark.sql.shuffle.partitions or adding a broadcast join to shrink the shuffle.',
+    genericRecommendation: () => 'Raise spark.sql.shuffle.partitions, or use a broadcast join for the smaller side.',
   },
   partitionSizing: {
     name: 'partition sizing',
     tag: 'PART',
-    thresholdSummary: () => 'partition byte size outside the configured target range',
+    thresholdSummary: (t) => `a shuffle partition over ${t.skewRatio}× the median or over ${t.maxPartBytes / 1073741824} GiB, or ${t.lowParTotalBytes / 1073741824} GiB of shuffle on ${t.lowParMaxTasks} tasks or fewer`,
     actionLabel(f) {
       switch (f.rule) {
         case 'shufflePartitionSkew': return 'Fix skewed partition';
@@ -121,8 +145,10 @@ export const FINDING_PRESENTATION
     },
     genericRecommendation(f) {
       switch (f.rule) {
-        case 'shufflePartitionSkew': return 'For join skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.';
-        case 'lowShuffleParallelism': return 'Raise spark.sql.shuffle.partitions so each partition is smaller.';
+        case 'shufflePartitionSkew': return skewJoinGeneric(f, 'For join skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.');
+        case 'lowShuffleParallelism': return switchAlreadyOn(f, 'spark.sql.shuffle.partitions')
+          ? "spark.sql.shuffle.partitions is already high enough, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller."
+          : 'Raise spark.sql.shuffle.partitions so each partition is smaller.';
         case 'maxPartitionTooBig': return 'Repartition to break up the oversized partition before this stage.';
       }
       return undefined;
@@ -141,7 +167,7 @@ export const FINDING_PRESENTATION
   gc: {
     name: 'GC pressure',
     tag: 'GC',
-    thresholdSummary: () => 'JVM GC time share above the configured ratio',
+    thresholdSummary: (t) => `GC above ${t.warnPct100}% (or below ${t.lowInfoPct100}%) of executor run time`,
     actionLabel: (f) => (f.direction === 'low' ? 'Right-size executor memory' : 'Reduce GC pressure'),
     genericRecommendation: (f) => (f.direction === 'low'
       ? 'Memory may be over-provisioned here: consider reducing spark.executor.memory for cost savings.'
@@ -158,14 +184,14 @@ export const FINDING_PRESENTATION
   failures: {
     name: 'failed tasks',
     tag: 'FAIL',
-    thresholdSummary: () => 'task failures above the configured rate',
+    thresholdSummary: (t) => `over ${shareLabel(t.warnRate)} of a stage's tasks failing`,
     actionLabel: () => 'Investigate task failures',
     genericRecommendation: () => 'Investigate driver logs for executor instability or data-driven errors.',
   },
   retryWaste: {
     name: 'retry waste',
     tag: 'RETRY',
-    thresholdSummary: () => 'retried task attempts consuming executor time',
+    thresholdSummary: (t) => `${t.minWasted}+ retried attempts wasting at least ${t.minWasteMs / 1000}s`,
     actionLabel: () => 'Investigate retry cause',
     genericRecommendation: () => 'Investigate executor loss or fetch failures behind the retried attempts.',
   },
@@ -173,7 +199,7 @@ export const FINDING_PRESENTATION
   slowHost: {
     name: 'slow executor host',
     tag: 'HOST',
-    thresholdSummary: (t) => `a host running ${t.ratioWarn}x+ slower than its peers by mean task duration (per-executor byte/time dimensions use a separate, narrower ratio ladder starting at ${t.ratioTiers[0]}x; only those can reach critical on ratio alone)`,
+    thresholdSummary: (t) => `a host ${t.ratioWarn}× slower than its peers by mean task time (per-executor figures from ${t.ratioTiers[0]}×)`,
     actionLabel(f) {
       if (f.variant === 'durationShare') return 'Fix data locality';
       if (f.variant === 'multiDim') return 'Investigate degraded executor';
@@ -182,7 +208,10 @@ export const FINDING_PRESENTATION
     genericRecommendation(f) {
       if (f.variant === 'durationShare') return 'Check for data locality or partition assignment skewing work onto one node.';
       if (f.variant === 'multiDim') return 'Investigate uneven partition assignment or a degraded executor.';
-      return 'Check what this host was running: it may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable spark.speculation to relaunch a lagging task automatically.';
+      const check = 'Check what this host was running: it may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault.';
+      return switchAlreadyOn(f, 'spark.speculation')
+        ? `${check} Speculation is already on, so a lagging task there is already relaunched.`
+        : `${check} Enable spark.speculation to relaunch a lagging task automatically.`;
     },
   },
   stageSlowness: {
@@ -202,14 +231,14 @@ export const FINDING_PRESENTATION
   speculationWaste: {
     name: 'speculation waste',
     tag: 'SPEC',
-    thresholdSummary: () => 'speculative task attempts that completed after the original',
+    thresholdSummary: (t) => `${t.minWasted}+ discarded speculative attempts wasting at least ${t.minWasteMs / 1000}s`,
     actionLabel: () => 'Tune speculation settings',
     genericRecommendation: () => 'If task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.',
   },
   coldStart: {
     name: 'cold start',
     tag: 'COLD',
-    thresholdSummary: () => 'executor startup time above the configured floor',
+    thresholdSummary: (t) => `the first stage waiting over ${t.gapSeconds}s for an executor`,
     actionLabel: () => 'Pre-warm cluster',
     genericRecommendation: () => 'Keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it does not scale up from zero.',
   },
@@ -230,7 +259,9 @@ export const FINDING_PRESENTATION
     },
     genericRecommendation(f) {
       switch (f.variant) {
-        case 'idleCores': return 'Reduce cluster size or enable dynamic allocation.';
+        case 'idleCores': return switchAlreadyOn(f, DYNAMIC_ALLOCATION_KEY)
+          ? 'Dynamic allocation is already on, so reduce cluster size.'
+          : 'Reduce cluster size or enable dynamic allocation.';
         case 'wasteModel': return 'Review spark.executor.memory and executor count.';
         case 'memoryBand':
           if (f.dataUnavailable) return undefined;
@@ -244,16 +275,18 @@ export const FINDING_PRESENTATION
   utilization: {
     name: 'executor utilization',
     tag: 'UTIL',
-    thresholdSummary: () => 'core occupancy below the configured floor across the run',
+    thresholdSummary: (t) => `average executor utilization below ${shareLabel(t.minUtil)}`,
     actionLabel: () => 'Reduce cluster size',
-    genericRecommendation: () => 'Consider reducing cluster size or enabling dynamic allocation.',
+    genericRecommendation: (f) => (switchAlreadyOn(f, DYNAMIC_ALLOCATION_KEY)
+      ? 'Dynamic allocation is already on, so consider reducing cluster size.'
+      : 'Consider reducing cluster size or enabling dynamic allocation.'),
   },
   coreLocality: {
     name: 'core locality',
     tag: 'LOCAL',
     thresholdSummary: () => 'task placement missing data-local core assignment',
     actionLabel: () => 'Fix data locality',
-    genericRecommendation: () => 'Check spark.locality.wait settings and executor/data colocation.',
+    genericRecommendation: () => 'Check executor/data colocation.',
   },
   cachingOpportunity: {
     name: 'caching opportunity',
@@ -280,14 +313,14 @@ export const FINDING_PRESENTATION
   jobFailureRate: {
     name: 'job failure rate',
     tag: 'JOBS',
-    thresholdSummary: () => 'job failure rate above the configured threshold',
+    thresholdSummary: (t) => `at least ${shareLabel(t.infoRate)} of jobs failing`,
     actionLabel: () => 'Investigate failed jobs',
     genericRecommendation: () => 'Inspect the driver log for the failed job(s) and the stage failures that triggered them.',
   },
   autoscalingChurn: {
     name: 'autoscaling churn',
     tag: 'CHRN',
-    thresholdSummary: () => 'executor add/remove churn above the configured rate',
+    thresholdSummary: (t) => `over ${shareLabel(t.warningPct)} of executors living under ${t.shortLivedMs / 60000} minutes`,
     actionLabel: () => 'Reduce autoscaling churn',
     genericRecommendation: () => 'This looks like wasteful re-provisioning rather than normal scale-down: consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.',
   },
@@ -305,7 +338,7 @@ export const FINDING_PRESENTATION
   smallFiles: {
     name: 'small files',
     tag: 'PLAN',
-    thresholdSummary: () => 'output files below the configured target size',
+    thresholdSummary: (t) => `over ${t.minFiles} files averaging under ${t.maxAvgFileSizeMB} MiB`,
     actionLabel: (f) => (f.direction === 'write' ? 'Coalesce output files' : 'Compact small files'),
     genericRecommendation: (f) => (f.direction === 'write'
       ? 'Repartition or coalesce before writing to raise the average file size.'
@@ -321,9 +354,11 @@ export const FINDING_PRESENTATION
   overBroadcast: {
     name: 'oversized broadcast join',
     tag: 'PLAN',
-    thresholdSummary: () => 'a broadcast join above the configured size ceiling',
+    thresholdSummary: (t) => `a broadcast over ${t.overBroadcastBytes / 1073741824} GiB`,
     actionLabel: () => 'Fix oversized broadcast',
-    genericRecommendation: () => 'Check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.',
+    genericRecommendation: (f) => (switchAlreadyOn(f, 'spark.sql.autoBroadcastJoinThreshold')
+      ? 'Automatic broadcast is already disabled, so remove the broadcast() hint that forced it.'
+      : 'Check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.'),
   },
 };
 

package/vendor-core/finding-tag-help.js

@@ -1,5 +1,5 @@
 // Plain-language help per finding tag: the expansion the tag stands for and a one-line description
-// of what it means. Shared by the dashboard (tag tooltips, verdict "What's happening") and the
+// of what it means. Shared by the dashboard (tag tooltips) and the
 // comparison verdict, which names finding categories by their expansion on every path.
                 
                     

package/vendor-core/finding-types.js

@@ -10,6 +10,16 @@
                                                                       
                                                           
 
+/** One concrete change a finding's fix makes, alongside its prose `recommendation`. Only a Spark
+ * property the detector already names; `suggested` is null when it computes no value. Sizes carry a
+ * Spark unit suffix ("384m"), counts are plain numbers, switches are booleans. */
+                              
+               
+              
+                                             
+                                              
+ 
+
 // The columns every finding carries, whatever its detector.
                                          
               
@@ -22,6 +32,8 @@
                          
                   
                           
+                                                                                               
+                              
                      
                       
                               

package/vendor-core/format-utils.js

@@ -159,9 +159,9 @@ const FINDING_METRIC_UNIT
   speculationWasteMs: 'ms', retryWasteMs: 'ms', taskDurationP50: 'ms',
   stageDurationMinutes: 'minutes',
   'P95/median': 'ratio', 'max/median': 'ratio', pRatio: 'ratio', oiRatio: 'ratio',
-  taskStageSkew: 'ratio', hostMeanRatio: 'ratio', execMaxMedianRatio: 'ratio',
+  hostMeanRatio: 'ratio', execMaxMedianRatio: 'ratio',
   gcPct: 'pct', failureRate: 'pct', stragglerShare: 'pct',
-  hostDurationShare: 'pctFraction',
+  hostDurationShare: 'pctFraction', taskStageSkew: 'pctFraction',
   taskCount: 'count', speculativeTasks: 'count', subtreeOccurrences: 'count',
 };
 
@@ -207,7 +207,8 @@ export function formatDuration(ms                    )         {
   if (!ms || ms <= 0) return '—';
   if (ms < 1000) return `${Math.round(ms)}ms`;
   if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
-  return `${Math.floor(ms / 60000)}m ${Math.floor((ms % 60000) / 1000)}s`;
+  if (ms < 3600000) return `${Math.floor(ms / 60000)}m ${Math.floor((ms % 60000) / 1000)}s`;
+  return `${Math.floor(ms / 3600000)}h ${Math.floor((ms % 3600000) / 60000)}m`;
 }
 
 export function recommendPartitions(stage                                                 )                                                {

package/vendor-core/impact-estimator.js

@@ -1,6 +1,6 @@
                                           
 import { ENTRY_BY_TYPE } from './detectors.js';
-                                                     
+import { coreTimeFor,                  } from './impact-model.js';
 
                                                      
 
@@ -9,7 +9,25 @@ import { ENTRY_BY_TYPE } from './detectors.js';
 export function estimateImpact(findings           , ctx             )            {
   for (const f of findings) {
     const estimate = ENTRY_BY_TYPE.get(f.type)?.estimate(f, ctx);
-    if (estimate) f.impactEstimate = estimate;
+    if (!estimate) continue;
+    f.impactEstimate = { ...estimate, coreTimeMs: coreTimeFor(f, estimate) };
   }
+  countTailCoreTimeOnce(findings);
   return findings;
 }
+
+// skew and straggler claim the same slow tail of a stage, so each reports its removed task time:
+// summed over a stage's findings that would count the tail twice. skew keeps the figure; a
+// straggler on the same stage carries null, since its tail is already counted there.
+const TAIL_CORE_TIME_ORDER = ['skew', 'straggler'];
+
+function countTailCoreTimeOnce(findings           )       {
+  const counted = new Set        ();
+  const tails = findings
+    .filter((f) => f.stageId != null && f.impactEstimate?.coreTimeMs != null && TAIL_CORE_TIME_ORDER.includes(f.type))
+    .sort((a, b) => TAIL_CORE_TIME_ORDER.indexOf(a.type) - TAIL_CORE_TIME_ORDER.indexOf(b.type));
+  for (const f of tails) {
+    if (counted.has(f.stageId          )) f.impactEstimate .coreTimeMs = null;
+    else counted.add(f.stageId          );
+  }
+}

package/vendor-core/impact-format.js

@@ -18,12 +18,13 @@ export function impactFigure(finding         )                {
   return null;
 }
 
-/** What a raw-waste figure counts, by its unit, as the words that follow it. */
-export function rawWasteMeaning(unit                                    )                {
-  switch (unit) {
+/** What a raw-waste figure counts, by its unit, as the words that follow it: an idle core
+ * figure is capacity no task ran on, not core time. */
+export function rawWasteMeaning(rawWaste                            )                {
+  switch (rawWaste?.unit) {
     case 'mbSeconds': return 'of unused executor memory';
     case 'coreHours':
-    case 'coreMs': return 'of core time';
+    case 'coreMs': return rawWaste.idle ? 'of idle core capacity' : 'of core time';
     case 'bytes': return 'of extra data written';
     case 'ms': return 'of task time';
     default: return null;
@@ -39,7 +40,7 @@ export function savingsMeaning(finding         )                {
   const estimate = finding.impactEstimate;
   if (!estimate) return null;
   if (estimate.wallClock) return 'of run time';
-  return rawWasteMeaning(estimate.rawWaste?.unit);
+  return rawWasteMeaning(estimate.rawWaste);
 }
 
 /** A finding's "Potential savings" figure as the widget board shows it: the
@@ -54,7 +55,7 @@ export function impactEstimateFigure(estimate                            )
     return { text: formatWallClockRange(estimate.wallClock .low, estimate.wallClock .high), meaning: 'of run time' };
   }
   const rawWasteText = estimate.rawWaste && estimate.rawWaste.value > 0 ? formatRawWaste(estimate.rawWaste) : null;
-  if (rawWasteText && !readsAsZero(rawWasteText)) return { text: rawWasteText, meaning: rawWasteMeaning(estimate.rawWaste .unit) };
+  if (rawWasteText && !readsAsZero(rawWasteText)) return { text: rawWasteText, meaning: rawWasteMeaning(estimate.rawWaste) };
   return null;
 }
 
@@ -84,16 +85,17 @@ export function impactEstimateCompact(estimate                            )
  * and the raw waste behind it. Null when the finding carries no estimate
  * model (`estimateMethod: 'none'`), no estimate at all, or a figure that
  * reads as zero (the step shows no savings then either). Uses the same
- * formatting and zero rules as the step's own savings figure. */
+ * formatting and zero rules as the step's own savings figure, which it does
+ * not repeat: the step already shows it. */
 export function estimateProvenance(finding                                 )                {
   const estimate = finding.impactEstimate;
   if (!estimate || estimate.estimateMethod === 'none') return null;
-  const method = estimate.estimateMethod;
+  const method = `${estimate.estimateMethod[0].toUpperCase()}${estimate.estimateMethod.slice(1)}`;
   const rawWaste = estimate.rawWaste && estimate.rawWaste.value > 0 ? estimate.rawWaste : null;
   const raw = rawWaste && !readsAsZero(formatRawWaste(rawWaste)) ? formatRawWaste(rawWaste) : null;
   const wallClock = estimate.wallClock;
   if (estimate.basis === 'resourceOnly') {
-    return raw ? `No run-time claim, ${method}. ${raw} was wasted, but it may not shorten the run.` : null;
+    return raw ? `${method}; ${raw} wasted, which may not shorten the run.` : null;
   }
   if (!wallClock || wallClock.high <= 0) return null;
   const highText = formatWallClockRange(wallClock.high, wallClock.high);
@@ -102,13 +104,12 @@ export function estimateProvenance(finding                                 )
   if (raw && rawWaste .unit !== 'ms') rawNote = ` Resource waste measured: ${raw}.`;
   else if (raw && rawWaste .value > wallClock.high && raw !== highText) rawNote = ` Raw waste before the floor clipped it: ${raw}.`;
   if (estimate.basis === 'serial') {
-    return `${highText}, ${method}. The stage ran effectively alone, so this is close to a point estimate.${rawNote}`;
+    return `${method}; the stage ran alone, so this is close to a point estimate.${rawNote}`;
   }
   if (estimate.basis === 'contended') {
     const lowText = formatWallClockRange(wallClock.low, wallClock.low);
-    const range = formatWallClockRange(wallClock.low, wallClock.high);
-    const spread = lowText === highText ? 'its floor and optimistic high agree' : `${lowText} is the floor, ${highText} assumes the fix fully lands`;
-    return `${range}, ${method}. The stage shared the cluster with others: ${spread}.${rawNote}`;
+    const spread = lowText === highText ? 'its floor and high agree' : `${lowText} is the floor, ${highText} if the fix fully lands`;
+    return `${method}; the stage shared the cluster: ${spread}.${rawNote}`;
   }
   return null;
 }

package/vendor-core/impact-model.js

@@ -1,7 +1,8 @@
 // The waste models every detector entry's estimate() builds its ImpactEstimate from: the assumed
 // throughputs, the per-stage measurements behind them and the occupancy clip wrappers. Each
 // finding type's own composition of these lives on its DETECTORS entry, next to its detect().
-                                                                                              
+                                                                                                                     
+import { isPythonStage } from './python-stage.js';
 import { nsToMs } from './format-utils.js';
 import {
   estimateSingleStage, estimateMultiStage,
@@ -16,6 +17,8 @@ import {
                                              
                                                                                  
                      
+                                                                                                  
+                                  
  
 
 // Assumed shuffle-network throughput per executor link, ~1 Gbps. Starting assumption, unvalidated.
@@ -104,13 +107,14 @@ const IDLE_CPU_SHARE_MAX = 0.01;
 
 // True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
 // the share can't be trusted: no CPU time recorded (older Spark logs omit the metric), or Python
-// code run through PythonRDD, whose worker-process CPU executorCpuTime (the JVM task thread's)
-// never counts (such stages read 0.1% on the same logs while computing).
-export function tasksMostlyIdle(stage       )          {
+// code run through a Python worker (isPythonStage: a PythonRDD stage or a Python UDF operator in
+// its SQL plan), whose worker-process CPU executorCpuTime (the JVM task thread's) never counts
+// (such stages read 0.1% on the same logs while computing).
+export function tasksMostlyIdle(stage       , sql                            = new Map())          {
   const runMs = stage.executorRunTime ?? 0;
   const cpuMs = nsToMs(stage.executorCpuTime ?? 0);
   if (runMs <= 0 || cpuMs <= 0) return false;
-  if (/PythonRDD/.test(stage.name ?? '') || /org\.apache\.spark\.api\.python\./.test(stage.details ?? '')) return false;
+  if (isPythonStage(stage, sql)) return false;
   return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
 }
 
@@ -173,3 +177,21 @@ export function stageMappableWasteOrCostOnly(
   // null: every stage excluded from the sweep
   return multiStageImpact(stageIds, wasteMsByStage, ctx, 'modeled', rawWaste) ?? costOnly('modeled', rawWaste);
 }
+
+// Finding types whose raw figure is busy core time read straight from the log: gc's jvmGCTime
+// (coreMs) and the discarded speculative or retried attempts' run time ('ms' cross-task sums).
+// Any other core figure is idle capacity, or modeled on an assumed constant (coreLocality's
+// per-task fetch penalty, autoscalingChurn's executor-hours, jobFailureRate's job-hours).
+const MEASURED_CORE_TIME_FIGURE                      = new Set(['gc', 'retryWaste', 'speculationWaste']);
+
+/** The busy core time a finding's fix removes, in core-milliseconds, or null when the detector
+ * measures none. Only a measured figure counts: one its estimate() already set (skew and
+ * straggler's removed task time), or the raw figure of a MEASURED_CORE_TIME_FIGURE type. A
+ * wall-clock claim, an idle capacity figure and a modeled figure are never converted. It never
+ * reads executorCpuTime, which leaves out Python worker CPU. */
+export function coreTimeFor(finding         , estimate                )                                       {
+  if (estimate.coreTimeMs !== undefined) return estimate.coreTimeMs;
+  const raw = estimate.rawWaste;
+  if (!raw || !MEASURED_CORE_TIME_FIGURE.has(finding.type)) return null;
+  return { low: raw.value, high: raw.value };
+}

package/vendor-core/ingest.js

@@ -14,6 +14,7 @@
                                             
                                                    
                                                     
+                                                   
                                    
                                     
  
@@ -39,9 +40,10 @@ export function routeMessage(
     case 'runAggregates': handlers.onRunAggregates?.(data.data); break;
     case 'stageExecutorMetrics': handlers.onStageExecutorMetrics?.(data.data); break;
     case 'stageSpeculationWaste': handlers.onStageSpeculationWaste?.(data.data); break;
+    case 'stageLateAttemptWork': handlers.onStageLateAttemptWork?.(data.data); break;
     case 'done': {
-      const { skippedLines } = data;
-      handlers.onDone?.({ skippedLines });
+      const { skippedLines, unreadableSqlExecutions } = data;
+      handlers.onDone?.(unreadableSqlExecutions ? { skippedLines, unreadableSqlExecutions } : { skippedLines });
       break;
     }
     case 'error':    handlers.onError?.(data); break;

package/vendor-core/list-runs.js

@@ -5,6 +5,7 @@ import { reassembleRollingEntries } from './parser-worker.js';
 import { peekLogHeader } from './log-header-peek.js';
 import { mcpError } from './mcp-error.js';
 import { normalizeBaseUrl } from './shs-request.js';
+import { isConnectionFailure } from './proxy.js';
 import { DEFAULT_IDLE_TIMEOUT_MS, DEFAULT_MAX_ARCHIVE_BYTES } from './shs-load.js';
 
                                
@@ -205,10 +206,12 @@ export async function listRunsShs(
   let res          ;
   try {
     // An unresponsive SHS would otherwise hang the tool call forever. A timed-out signal rejects
-    // the fetch with an AbortError, which this same catch turns into access-or-upstream-failure.
+    // the fetch with a TimeoutError, which this same catch turns into upstream-unreachable, the
+    // code a refused connection gets too.
     res = await fetchImpl(url.toString(), { signal: AbortSignal.timeout(DEFAULT_IDLE_TIMEOUT_MS) });
   } catch (e) {
-    throw mcpError('access-or-upstream-failure', `Could not reach ${normalized}: ${e instanceof Error ? e.message : String(e)}`);
+    const code = isConnectionFailure(e) ? 'upstream-unreachable' : 'access-or-upstream-failure';
+    throw mcpError(code, `Could not reach ${normalized}: ${e instanceof Error ? e.message : String(e)}`);
   }
   if (!res.ok) {
     throw mcpError('access-or-upstream-failure', `SHS applications list request failed with status ${res.status}.`);

package/vendor-core/mcp-tools.js

@@ -52,7 +52,7 @@ import { typeTag } from './format-utils.js';
                                        
                  
                  
-                                                                               
+                                                                                      
                                  
                                                
                                              

package/vendor-core/model-assembler.js

@@ -2,6 +2,7 @@
            
                
         
+                     
                
            
                 
@@ -9,6 +10,15 @@
                 
                     
 
+// The done message's counts of what the parse could not read. On the model, not only passed to onDone,
+// so every consumer of the model (dashboard export, CLI, MCP) reports them the same way.
+function recordParseGaps(appModel          , data         )       {
+  const done = data                                                                             ;
+  appModel.skippedLines = done?.skippedLines ?? 0;
+  if (done?.unreadableSqlExecutions) appModel.unreadableSqlExecutions = done.unreadableSqlExecutions;
+  else delete appModel.unreadableSqlExecutions;
+}
+
 // Worker-message -> appModel assembly. Shared by the file-load and SHS-URL-load paths. Pure model
 // mutation: analysis/render/persist stay in the caller via the onDone/onProgress/onError hooks.
 export function createModelCallbacks(
@@ -75,6 +85,18 @@ export function createModelCallbacks(
         }
       }
     },
-    onDone, onError,
+    onDone(data         ) {
+      recordParseGaps(appModel, data);
+      onDone?.(data);
+    },
+    // Patch the late work of failed attempts, whose TaskEnds arrive after StageCompleted.
+    // `data` is Map<stageId, StageAttemptTotals>.
+    onStageLateAttemptWork(data         ) {
+      for (const [stageId, work] of data                                   ) {
+        const stage = appModel.stages.get(stageId);
+        if (stage) stage.lateAttemptWork = work;
+      }
+    },
+    onError,
   };
 }

package/vendor-core/parser-worker.js

@@ -45,7 +45,7 @@ const NOT_AN_EVENT_LOG = 'Not a Spark event log: no application-start event foun
 // Minimal shape streamFile/runParse/runParseFiles read off `file` (name, size,
 // slice(start,end).arrayBuffer()): narrower than the full DOM `File`. A real
 // `File` (the browser Worker path) satisfies it structurally, but so does the
-// plain object src/cli/collect-run.ts's nodeFileFromPath builds for Node, which
+// plain object packages/core/src/cli/collect-run.ts's nodeFileFromPath builds for Node, which
 // has no DOM `File` constructor.
                    
                

package/vendor-core/proxy.js

@@ -10,7 +10,9 @@ function sendSafeError(res, status, code) {
   res.end(body);
 }
 
-function isConnectionFailure(error) {
+// Also read by list-runs.ts, so a History Server listing that can't be reached reports the
+// same upstream-unreachable code as a run fetch.
+export function isConnectionFailure(error) {
   if (error?.name === 'AbortError' || error?.name === 'TimeoutError') return true;
   const code = error?.code ?? error?.cause?.code;
   if (['ECONNREFUSED', 'ECONNRESET', 'EHOSTUNREACH', 'ENETUNREACH', 'ENOTFOUND', 'ETIMEDOUT'].includes(code)) return true;

package/vendor-core/python-stage.js

@@ -0,0 +1,25 @@
+import { planNodesOfStage } from './stage-plan-nodes.js';
+                                                      
+
+// Plan operators that hand rows to a Python worker process: the row-at-a-time and Arrow Python UDF
+// evaluators, the pandas/Arrow grouped and map operators, and the PythonRDD scan of an RDD
+// pipeline. A suffixed name is a variant of the same operator (BatchEvalPythonUDTF,
+// FlatMapGroupsInPandasWithState). Spark 4.1 renamed AggregateInPandas and WindowInPandas to
+// ArrowAggregatePython and ArrowWindowPython. Spark prefixes a whole-stage-codegen child's name
+// with "*(n) " in some plan strings.
+const PYTHON_PLAN_NODE = /^(?:\*\(\d+\)\s*)?(?:PythonRDD|BatchEvalPython|ArrowEvalPython|ArrowAggregatePython|ArrowWindowPython|\w+InPandas|\w+InArrow)\w*\b/;
+
+// An RDD lambda or map function has no SQL plan to match, and a stage whose plan could not be
+// matched is left with nothing but its name and call site.
+const PYTHON_STAGE_NAME = /PythonRDD/;
+const PYTHON_STAGE_DETAILS = /org\.apache\.spark\.api\.python\./;
+
+/** True when the stage ran Python code in a worker process: the union of a Python operator among
+ * the plan nodes attributed to it (catches Python UDFs inside SQL) and the stage's own name or
+ * call site naming PythonRDD / org.apache.spark.api.python (catches RDD lambdas, which have no
+ * plan, and stages that cannot be matched to one). The executor CPU time of such a stage misses
+ * the worker process's CPU. */
+export function isPythonStage(stage       , sql                           )          {
+  if (PYTHON_STAGE_NAME.test(stage.name ?? '') || PYTHON_STAGE_DETAILS.test(stage.details ?? '')) return true;
+  return planNodesOfStage(stage, sql).some((node) => PYTHON_PLAN_NODE.test(node.name ?? ''));
+}