sparkforensics-mcp 0.4.0 → 0.5.0

package/vendor-core/detectors.js

@@ -1,4 +1,5 @@
-import { pathBasename, formatBytes, nsToMs, IMPACT_BAND_ORDER } from './format-utils.js';
+import { pathBasename, formatBytes, IMPACT_BAND_ORDER } from './format-utils.js';
+import { shareLabel } from './finding-presentation.js';
 import { scanRelationId } from './plan-summary.js';
 import { computePeakConcurrentCores, computePeakConcurrentExecutorCount } from './core-count.js';
 import { walkPlanTree } from './plan-tree-walk.js';
@@ -13,11 +14,14 @@ import {
                    
 } from './impact-model.js';
 import { isExchangeNode, isBroadcastExchangeNode } from './plan-node-detail.js';
+import { totalExecutorCpuMs } from './run-totals.js';
+import { DUPLICATE_SUBTREE_DIFFERING_NOTE, duplicateSubtreeDetail, SLOW_HOST_DIMENSION_LABEL } from './finding-generic-recommendation.js';
 import { stageIdsForSqlExec } from './sql-stages.js';
 import { cyrb53 } from './string-hash.js';
+import { decreaseConf, increaseConf, setConf } from './remediation.js';
 import { MAX_FAILURE_GROUPS, describeTaskFailure,                       } from './task-failure.js';
                                                                                                
-                                                                                                         
+                                                                                                                      
 
 const MB = 1024 * 1024;
 const GB = 1024 * MB;
@@ -637,10 +641,12 @@ function stragglerTailClaim(stage           )            {
   return tailClaim(stage, Math.max(0, (stage.taskDurationMax ?? 0) - longestTaskAfterFixMs), longestTaskAfterFixMs);
 }
 
-// A tail claim shortens the stage's longest task, hence TAIL_CLAIM (see occupancy.ts).
+// A tail claim shortens the stage's longest task, hence TAIL_CLAIM (see occupancy.ts). Its core
+// time is the measured task time the fix removes (removedCoreWorkMs).
 function tailClaimImpact(claim           , stageId        , ctx             )                 {
-  return singleStageImpact(claim.wasteMs, stageId, ctx, 'measured', { value: claim.wasteMs, unit: 'ms' },
+  const estimate = singleStageImpact(claim.wasteMs, stageId, ctx, 'measured', { value: claim.wasteMs, unit: 'ms' },
     { ...TAIL_CLAIM, removedCoreWorkMs: claim.removedCoreWorkMs, longestTaskAfterFixMs: claim.longestTaskAfterFixMs });
+  return { ...estimate, coreTimeMs: { low: claim.removedCoreWorkMs, high: claim.removedCoreWorkMs } };
 }
 
 // The figure a runtime floor checks: the claim's recoverable wall-clock, not a delta a physical
@@ -649,6 +655,76 @@ function tailClaimFloorMs(claim           , stageId        , ctx             )
   return tailClaimImpact(claim, stageId, ctx.impact).wallClock?.high ?? claim.wasteMs;
 }
 
+// A setting is only a fix when the run's logged conf doesn't already have it: a run that set it
+// needs another remedy. Only explicitly logged properties count; Spark's unlogged version
+// defaults are not modeled. Booleans compare case-insensitively, as Spark parses them.
+function loggedAs(app                    , key        , suggested                  )          {
+  const logged = app?.config?.[key]?.trim();
+  return typeof suggested === 'boolean'
+    ? logged?.toLowerCase() === String(suggested)
+    : logged === suggested;
+}
+
+function setConfUnlessLogged(app                    , key        , suggested                  )                {
+  return loggedAs(app, key, suggested) ? [] : [setConf(key, suggested)];
+}
+
+// A switch's fix worded for the run's logged conf: `recommend` names the property while the run
+// doesn't have it, `alreadyOn` points at the remedy left once it does, with no remediation.
+function switchFix(on         , key        , suggested                  , recommend        , alreadyOn        )                                               {
+  return on ? { text: alreadyOn, remediation: [] } : { text: recommend, remediation: [setConf(key, suggested)] };
+}
+
+function skewJoinFix(app                    )                                               {
+  const key = 'spark.sql.adaptive.skewJoin.enabled';
+  const remedy = 'salt the key or repartition on a better key';
+  if (loggedAs(app, 'spark.sql.adaptive.enabled', false)) {
+    return {
+      text: `AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise ${remedy}`,
+      remediation: [setConf('spark.sql.adaptive.enabled', true), ...setConfUnlessLogged(app, key, true)],
+    };
+  }
+  return switchFix(loggedAs(app, key, true), key, true,
+    `for join-driven skew, enable AQE skew-join handling (${key}); otherwise ${remedy}`,
+    `AQE skew-join handling is already on, so ${remedy}`);
+}
+
+// The resources flag is read from the same property as the logged conf.
+function dynamicAllocationFix(app             , recommend        , alreadyOn        )                                               {
+  const key = 'spark.dynamicAllocation.enabled';
+  return switchFix(app.resources?.dynamicAllocationEnabled === true || loggedAs(app, key, true), key, true, recommend, alreadyOn);
+}
+
+// The run's logged spark.sql.shuffle.partitions as a count, or null when unlogged or not a count.
+function loggedShufflePartitions(app                    )                {
+  const logged = app?.config?.['spark.sql.shuffle.partitions']?.trim();
+  return logged != null && /^\d+$/.test(logged) ? Number(logged) : null;
+}
+
+// lowShuffleParallelism's fix. The partition count that brings each shuffle partition down to the
+// ideal size, as the estimate models it, is per stage; the property is job-wide. A logged value at
+// or above it means the property is not what limits that stage (a repartition(n) or an RDD
+// shuffle is), so the text points at the stage's own partitioning and no property is suggested.
+// Unlogged, the count is not a safe value: null.
+function lowShuffleParallelismFix(app                    , needed        )                                               {
+  const logged = loggedShufflePartitions(app);
+  if (logged != null && logged >= needed) {
+    return {
+      text: `spark.sql.shuffle.partitions is already ${logged}, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller`,
+      remediation: [],
+    };
+  }
+  return {
+    text: 'raise spark.sql.shuffle.partitions so each partition is smaller',
+    remediation: [increaseConf('spark.sql.shuffle.partitions', logged == null ? null : needed)],
+  };
+}
+
+// No dynamic-allocation property has an effect on a run whose logged conf turns it off.
+function dynamicAllocationOff(app                    )          {
+  return app?.resources?.dynamicAllocationEnabled === false || app?.config?.['spark.dynamicAllocation.enabled']?.trim().toLowerCase() === 'false';
+}
+
 // Shared by cacheUtilization's two variants, worded per storage source: neither is a runtime
 // block-access read-count.
 const CACHE_UTILIZATION_VALIDATION = {
@@ -669,46 +745,53 @@ function cacheSampleConfidence(numPartitions        )
   return 'medium';
 }
 
+/** A sentence names a cached RDD by its first 40 characters, since RDD names are often a whole
+ * plan string (the Cache Storage card shows it in full); an unnamed RDD reads "RDD <id>". */
+function rddLabel({ id, name }                 )         {
+  return `RDD ${!name ? id : name.length > 40 ? `${name.slice(0, 40)}...` : name}`;
+}
+
 function partialCacheFinding(rdd                 , cachedRatio        , impactBand                    )          {
-  const rddName = rdd.name || `RDD ${rdd.id}`;
   const cachedPct = Math.round(cachedRatio * 100);
   const evictedPct = 100 - cachedPct;
   return {
     type: 'cacheUtilization', variant: 'partialCache', stageId: null,
-    rddId: rdd.id, rddName,
+    rddId: rdd.id, rddName: rdd.name || `RDD ${rdd.id}`,
     impactBand, metric: 'cachedRatio', value: cachedPct,
     confidence: cacheSampleConfidence(rdd.numPartitions),
     validationRequired: CACHE_UTILIZATION_VALIDATION[rdd.storageSource ?? 'rddInfo'],
     memorySize: rdd.memorySize, diskSize: rdd.diskSize,
     numCachedPartitions: rdd.numCachedPartitions, numPartitions: rdd.numPartitions,
-    recommendation: `RDD ${rddName} is ${evictedPct}% evicted from cache (${cachedPct}% of partitions cached). Increase executor memory or reduce the cached dataset size.`,
+    recommendation: `${rddLabel(rdd)} is ${evictedPct}% evicted from cache (${cachedPct}% of partitions cached): increase executor memory or reduce the cached dataset size.`,
+    remediation: [increaseConf('spark.executor.memory')],
   };
 }
 
 function diskSpilloverFinding(rdd                 , diskRatio        , impactBand                    )          {
-  const rddName = rdd.name || `RDD ${rdd.id}`;
   const diskPct = Math.round(diskRatio * 100);
   return {
     type: 'cacheUtilization', variant: 'diskSpillover', stageId: null,
-    rddId: rdd.id, rddName,
+    rddId: rdd.id, rddName: rdd.name || `RDD ${rdd.id}`,
     impactBand, metric: 'diskRatio', value: diskPct,
     confidence: cacheSampleConfidence(rdd.numPartitions),
     validationRequired: CACHE_UTILIZATION_VALIDATION[rdd.storageSource ?? 'rddInfo'],
     memorySize: rdd.memorySize, diskSize: rdd.diskSize,
     numCachedPartitions: rdd.numCachedPartitions, numPartitions: rdd.numPartitions,
-    recommendation: `RDD ${rddName} is ${diskPct}% spilled to disk despite requesting MEMORY_AND_DISK. Executor memory may be too small for this cached dataset.`,
+    recommendation: `${rddLabel(rdd)} is ${diskPct}% spilled to disk despite requesting MEMORY_AND_DISK: executor memory may be too small for it.`,
+    remediation: [increaseConf('spark.executor.memory')],
   };
 }
 
 // Persisted RDDs with no storage evidence at all: no block updates in the log, and RDD Info's
 // sizes are the 0 that Spark 2.3+ always writes. Reports the gap instead of a clean result, the
 // same missing-evidence shape as memoryUtilization's dataUnavailable caveat.
-function storageUnobservedFinding(persistedRddCount        )          {
+function storageUnobservedFinding(persistedRddCount        , app                    )          {
   const rdds = persistedRddCount === 1 ? '1 persisted RDD has' : `${persistedRddCount} persisted RDDs have`;
   return {
     type: 'cacheUtilization', variant: 'storageUnobserved', stageId: null,
     impactBand: 'info', metric: 'persistedRdds', value: persistedRddCount, dataUnavailable: true,
     recommendation: `${rdds} no cache-storage evidence in this log, so eviction and disk spillover can't be checked: Spark 2.3+ records cached sizes only as block updates, which need spark.eventLog.logBlockUpdates.enabled=true.`,
+    remediation: setConfUnlessLogged(app, 'spark.eventLog.logBlockUpdates.enabled', true),
   };
 }
 
@@ -955,19 +1038,13 @@ function cachingReuseConfidence(occurrences        , minExecutions        )
   return 'medium';
 }
 
-// A share threshold as caveat text states it: 0.005 -> "0.5%". Rounded to 4 decimals of a percent
-// so float noise (0.07 * 100) never prints.
-function shareLabel(share        )         {
-  return `${Math.round(share * 1e6) / 1e4}%`;
-}
-
 // Caveats that name a threshold read it from the thresholds the detector ran with, so a tuned run
 // states the floor it actually used.
 function gcValidation(minRunTimeMs        )         {
-  return `This finding is gated by a ${minRunTimeMs / 1000}-second minimum-runtime floor, our own noise floor for this metric.`;
+  return `Checked only on stages with at least ${minRunTimeMs / 1000}s of executor run time.`;
 }
 
-const INCOMPLETE_RUN_RECOMMENDATION = 'This event log never recorded an ApplicationEnd event: the capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Findings and metrics elsewhere on this board reflect only what was captured up to that point, not the full run.';
+const INCOMPLETE_RUN_RECOMMENDATION = 'This event log never recorded an ApplicationEnd event: the capture stopped before the run finished (a job still running, a rotated log, or a cut-short capture), so every figure on this board covers only what was captured.';
 
 export const DETECTORS = [
   defineStageDetector({
@@ -984,13 +1061,15 @@ export const DETECTORS = [
       const floorWasteMs = tailClaimFloorMs(skewTailClaim(stage, metric === 'P95/median'), stage.id, ctx);
       if (!meetsRuntimeFloor(floorWasteMs, appDurationMs(ctx.app), thresholds.floorPctWarn)) return null;
       const value = Math.round(ratio * 10) / 10;
+      const fix = skewJoinFix(ctx.app);
       return {
         type: 'skew', stageId: stage.id,
         impactBand: 'warning',
         metric, value,
         confidence: skewConfidence(ratio, thresholds.ratioWarn),
-        validationRequired: `This finding is gated by a ${shareLabel(thresholds.floorPctWarn)} runtime-floor threshold, our own noise floor for this metric.`,
-        recommendation: `Task duration ratio (${metric}) is ${value}×: for join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key to reduce task skew.`,
+        validationRequired: `Flagged only when it costs at least ${shareLabel(thresholds.floorPctWarn)} of run time.`,
+        recommendation: `Task duration ratio (${metric}) is ${value}×: ${fix.text}.`,
+        remediation: fix.remediation,
       };
     },
     estimate(finding, ctx)                        {
@@ -1002,14 +1081,22 @@ export const DETECTORS = [
     },
   }),
   defineStageDetector({
-    type: 'stageShape', order: 35, fixEffort: 'code', version: 1,
+    type: 'stageShape', order: 35, fixEffort: 'code', version: 2,
     emits: ['stageShape'],
     docAnchor: '#bottleneck-stage-shape',
     // lowParallelismFloorPct: the same 0.5% runtime floor the tiered detectors use. Parallelizing a
     // stage can't save more than the stage's own duration, so a shorter stage can't clear it; on
     // the 14 real logs that was 2839 of 3005 lowParallelism findings (2168 on sub-second stages).
     // App-wide idle capacity stays covered by utilization.
-    thresholds: { pRatioMax: 0.5, oiRatioMax: 10, skewWarn: 3, lowParallelismFloorPct: 0.005 },
+    // taskStageSkew: stageShareMin is the share of the stage's wall-clock the longest task must
+    // span, and skewWarn how far past the median task it must run (skew's own max/median 3×).
+    // The share alone can't tell a straggler apart: on a single wave (tasks <= cores) the longest
+    // task spans nearly the whole stage however even the tasks are. taskStageSkewFloorPct is the
+    // same 0.5% runtime floor as lowParallelismFloorPct.
+    thresholds: {
+      pRatioMax: 0.5, oiRatioMax: 10, skewWarn: 3, stageShareMin: 0.5,
+      lowParallelismFloorPct: 0.005, taskStageSkewFloorPct: 0.005,
+    },
     detect(stage, ctx, thresholds)            {
       const out            = [];
       const execCount = (stage.executorStats ?? []).length;
@@ -1025,7 +1112,7 @@ export const DETECTORS = [
             rule: 'lowParallelism', metric: 'pRatio', value: Math.round(pRatio * 100) / 100,
             // Absolute core count behind pRatio, for the impact estimator's idle-core-ms figure.
             totalCores,
-            recommendation: `This stage runs ${stage.taskCount} ${stage.taskCount === 1 ? 'task' : 'tasks'} across ~${totalCores} cores, so it is under-parallelized and leaves cluster capacity idle.`,
+            recommendation: `This stage runs ${stage.taskCount} ${stage.taskCount === 1 ? 'task' : 'tasks'} across ~${totalCores} cores: it is under-parallelized and leaves cluster capacity idle.`,
           });
         }
       }
@@ -1040,18 +1127,21 @@ export const DETECTORS = [
           });
         }
       }
-      // TaskStageSkew: straggler cost vs stage wall-clock. Skip near-zero duration. Always info
-      // like its siblings: this trigger forces the occupancy-clipped estimate to exactly zero on
-      // every firing, so there's no wall-clock-backed tier left to gate on.
-      if (stageDurationMs > 0) {
-        const ratio = stage.taskDurationMax / stageDurationMs;
-        if (ratio > thresholds.skewWarn) {
+      // TaskStageSkew: one straggler sets when the stage ends. A task runs inside its stage's
+      // window, so the longest task's share of the stage's wall-clock is at most 1. Skip a
+      // zero-length or single-task stage. Always info like its siblings: skew and straggler
+      // already make the wall-clock claim for the same tail, so this one reports idle core-time.
+      if (stageDurationMs > 0 && stage.taskCount > 1 && stage.taskDurationP50 > 0
+        && !stageBelowRuntimeFloor(stage, ctx, thresholds.taskStageSkewFloorPct)) {
+        const share = stage.taskDurationMax / stageDurationMs;
+        const vsMedian = stage.taskDurationMax / stage.taskDurationP50;
+        if (share > thresholds.stageShareMin && vsMedian > thresholds.skewWarn) {
           out.push({
             type: 'stageShape', stageId: stage.id, impactBand: 'info',
-            rule: 'taskStageSkew', metric: 'taskStageSkew', value: Math.round(ratio * 10) / 10,
+            rule: 'taskStageSkew', metric: 'taskStageSkew', value: Math.round(share * 100) / 100,
             // Absolute core count, for the impact estimator's idle-core-ms figure.
             totalCores,
-            recommendation: `One task takes ${Math.round(ratio * 10) / 10}× this stage's wall-clock duration; a single straggler is gating the whole stage.`,
+            recommendation: `The longest task ran for ${Math.round(share * 100)}% of this stage's wall-clock, ${Math.round(vsMedian * 10) / 10}× the median task: a single straggler is gating the whole stage.`,
           });
         }
       }
@@ -1065,7 +1155,7 @@ export const DETECTORS = [
         const idleCoreMs =
           Math.max(0, ((finding.totalCores                      ) ?? 0) - (stage.taskCount ?? 0)) * stageDurationMs;
         // Real per-stage data (cores, task count, duration), no assumed constant.
-        return costOnly('measured', { value: idleCoreMs, unit: 'coreMs' });
+        return costOnly('measured', { value: idleCoreMs, unit: 'coreMs', idle: true });
       }
       if (finding.rule === 'dataExplosion') {
         const excessBytes = Math.max(0, (stage.outputBytes ?? 0) - (stage.inputBytes ?? 0));
@@ -1076,12 +1166,12 @@ export const DETECTORS = [
         const totalCores = (finding.totalCores                      ) ?? 0;
         const taskCount = stage.taskCount ?? 0;
         // Cores idle during the straggler's tail, at achieved concurrency (not full cluster
-        // capacity, which is lowParallelism's territory): this rule's trigger forces the
-        // occupancy-clipped estimate to zero on every firing, so it's resourceOnly, not a wall-clock claim.
+        // capacity, which is lowParallelism's territory): resourceOnly, since skew and straggler
+        // already claim that tail's wall-clock time.
         const idleCoreMs =
           Math.max(0, Math.min(totalCores, taskCount) - 1) *
           Math.max(0, (stage.taskDurationMax ?? 0) - (stage.taskDurationP50 ?? 0));
-        return costOnly('measured', { value: idleCoreMs, unit: 'coreMs' });
+        return costOnly('measured', { value: idleCoreMs, unit: 'coreMs', idle: true });
       }
       return null;
     },
@@ -1103,6 +1193,7 @@ export const DETECTORS = [
         impactBand: 'info',
         metric: 'shuffleReadBytes', value: bytes,
         recommendation: `${formatBytes(bytes)} shuffled in this stage: consider increasing spark.sql.shuffle.partitions or adding a broadcast join.`,
+        remediation: [increaseConf('spark.sql.shuffle.partitions')],
       };
     },
     estimate(finding, ctx)                        {
@@ -1125,7 +1216,7 @@ export const DETECTORS = [
     emits: ['partitionSizing'],
     docAnchor: '#bottleneck-partition-sizing',
     thresholds: { skewRatio: 5, skewFloorBytes: 256 * MB, lowParTotalBytes: GB, lowParMaxTasks: 7, maxPartBytes: 5 * GB },
-    detect(stage, _ctx, thresholds)            {
+    detect(stage, ctx, thresholds)            {
       const out            = [];
       const { shuffleReadP50: p50, shuffleReadMax: max, shuffleReadBytes: total, taskCount } = stage;
       if (max > thresholds.skewRatio * p50 && max > thresholds.skewFloorBytes) {
@@ -1133,18 +1224,22 @@ export const DETECTORS = [
         // "Infinity×", so fall back to median-free phrasing.
         const ratioText = p50 > 0
           ? `${Math.round(max / p50 * 10) / 10}× the median (${formatBytes(p50)})`
-          : `far larger than the median (${formatBytes(p50)}, effectively empty)`;
+          : 'far larger than the median, which is effectively empty';
+        const fix = skewJoinFix(ctx.app);
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
           rule: 'shufflePartitionSkew', metric: 'shuffleReadMax', value: max,
-          recommendation: `The largest shuffle partition (${formatBytes(max)}) is ${ratioText}: for join skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.`,
+          recommendation: `The largest shuffle partition (${formatBytes(max)}) is ${ratioText}: ${fix.text}.`,
+          remediation: fix.remediation,
         });
       }
       if (total >= thresholds.lowParTotalBytes && taskCount <= thresholds.lowParMaxTasks) {
+        const fix = lowShuffleParallelismFix(ctx.app, Math.ceil(total / IDEAL_BYTES_PER_PARTITION_TASK));
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
           rule: 'lowShuffleParallelism', metric: 'taskCount', value: taskCount,
-          recommendation: `${Math.round(total / GB * 10) / 10} GB of shuffle spread over only ${taskCount} tasks: raise spark.sql.shuffle.partitions so each partition is smaller.`,
+          recommendation: `${Math.round(total / GB * 10) / 10} GB of shuffle spread over only ${taskCount} tasks: ${fix.text}.`,
+          remediation: fix.remediation,
         });
       }
       if (max >= thresholds.maxPartBytes) {
@@ -1212,6 +1307,7 @@ export const DETECTORS = [
         recommendation: cls === 'skew'
           ? `${formatBytes(stage.memoryBytesSpilled)} spilled, skew-driven: fix task skew first; adding memory will not help.`
           : `${formatBytes(stage.memoryBytesSpilled)} spilled: raise spark.sql.shuffle.partitions or increase executor memory.`,
+        remediation: cls === 'skew' ? [] : [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.executor.memory')],
       };
     },
     estimate(finding, ctx)                        {
@@ -1251,6 +1347,7 @@ export const DETECTORS = [
           metric: 'gcPct', value,
           confidence: gcConfidence(pct, thresholds, 'high'), validationRequired: gcValidation(thresholds.minRunTimeMs),
           recommendation: `GC consumed ${value}% of executor run time: reduce object creation, use primitive types, avoid UDFs, increase executor memory.`,
+          remediation: [increaseConf('spark.executor.memory')],
         };
       }
       // Low-GC (cost) branch: only for stages that ran long enough to be meaningful.
@@ -1264,6 +1361,7 @@ export const DETECTORS = [
           metric: 'gcPct', value,
           confidence: gcConfidence(pct, thresholds, 'low'), validationRequired: gcValidation(thresholds.minRunTimeMs),
           recommendation: `GC consumed only ${value}% of executor run time: memory may be over-provisioned; consider reducing spark.executor.memory for cost savings.`,
+          remediation: [decreaseConf('spark.executor.memory')],
         };
       }
       return null;
@@ -1315,6 +1413,9 @@ export const DETECTORS = [
       if ((hosts.length < thresholds.minHosts && execs0.length < thresholds.minHosts) || stage.taskCount < thresholds.minTasks) return null;
       if (stageBelowRuntimeFloor(stage, ctx, thresholds.stageFloorPct)) return null;
       const out            = [];
+      const speculation = switchFix(loggedAs(ctx.app, 'spark.speculation', true), 'spark.speculation', true,
+        'check what it was running, and consider enabling spark.speculation to relaunch a lagging task automatically',
+        'check what it was running; speculation is already on, so a lagging task there is already relaunched');
       if (hosts.length >= thresholds.minHosts) {
         const means = hosts.map(h => ({ host: h.host, taskCount: h.taskCount, mean: h.totalDuration / h.taskCount }));
         const sorted = [...means].map(h => h.mean).sort((a, b) => a - b);
@@ -1331,7 +1432,8 @@ export const DETECTORS = [
               // `value` is a ratio; the estimator needs the absolute per-host mean.
               hostMeanMs: h.mean,
               host: h.host, hostTaskShare: Math.round(share * 100) / 100,
-              recommendation: `${h.host} may just hold data locality for its tasks or carry one heavy stage, not necessarily a hardware fault: check what it was running, and consider enabling spark.speculation to relaunch a lagging task automatically.`,
+              recommendation: `${h.host} may just hold data locality for its tasks or carry one heavy stage, not necessarily a hardware fault: ${speculation.text}.`,
+              remediation: speculation.remediation,
             });
           }
         }
@@ -1387,7 +1489,7 @@ export const DETECTORS = [
           // in this dimension's own unit (ms for taskTime, bytes for the rest).
           execMaxValue: r.value,
           executorId: r.key,
-          recommendation: `Executor ${r.key} deviates ${Math.round(r.ratio * 10) / 10}× from the median on ${d.dimension}: investigate uneven partition assignment or a degraded executor.`,
+          recommendation: `Executor ${r.key}'s ${SLOW_HOST_DIMENSION_LABEL[d.dimension]} is ${Math.round(r.ratio * 10) / 10}× the median: investigate uneven partition assignment or a degraded executor.`,
         });
       }
       return out;
@@ -1434,6 +1536,7 @@ export const DETECTORS = [
         type: 'stageSlowness', stageId: stage.id, impactBand,
         metric: 'stageDurationMinutes', value,
         recommendation: `This stage ran ${value} minutes with no more specific cause flagged: often a partition-count problem, raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.`,
+        remediation: [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.default.parallelism')],
       };
     },
     estimate(finding, ctx)                        {
@@ -1454,7 +1557,7 @@ export const DETECTORS = [
         ? stage.taskActiveMs
         : Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
       const taskCount = stage.taskCount ?? 0;
-      const wasteMs = readBytes <= 0 && tasksMostlyIdle(stage) ? 0 : activeMs * Math.max(0, 1 - taskCount / ctx.totalCores);
+      const wasteMs = readBytes <= 0 && tasksMostlyIdle(stage, ctx.sql) ? 0 : activeMs * Math.max(0, 1 - taskCount / ctx.totalCores);
       return singleStageImpact(wasteMs, finding.stageId, ctx, 'modeled', { value: wasteMs, unit: 'ms' }, TAIL_CLAIM);
     },
   }),
@@ -1571,7 +1674,7 @@ export const DETECTORS = [
         confidence: useSpeculativeMetric
           ? stragglerConfidence(speculativeShare, thresholds.warnPct, thresholds.critPct)
           : stragglerConfidence(stragglerShare, thresholds.shareWarn, thresholds.critPct),
-        validationRequired: `This finding is gated by ${shareLabel(thresholds.floorPctWarn)}/${shareLabel(thresholds.floorPctCrit)} runtime-floor thresholds, our own noise floor for this metric.`,
+        validationRequired: `Warning needs at least ${shareLabel(thresholds.floorPctWarn)} of run time at stake, critical ${shareLabel(thresholds.floorPctCrit)}.`,
         recommendation: `${detail}: rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that's a candidate for AQE's skew-join handling.`,
       };
     },
@@ -1596,7 +1699,8 @@ export const DETECTORS = [
         impactBand: 'warning',
         metric: 'speculationWasteMs', value: wastedMs,
         confidence: speculationWasteConfidence(wastedMs, thresholds.minWasteMs),
-        recommendation: `Speculative execution discarded ${Math.round(wastedMs / 1000)}s of executor time in this stage; if task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.`,
+        recommendation: `Speculative execution discarded ${Math.round(wastedMs / 1000)}s of executor time in this stage: if task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.`,
+        remediation: [increaseConf('spark.speculation.multiplier'), increaseConf('spark.speculation.quantile')],
       };
     },
     estimate(finding, ctx)                        {
@@ -1658,6 +1762,7 @@ export const DETECTORS = [
         type: 'tinyTask', stageId: stage.id, impactBand: 'info',
         metric: 'taskDurationP50', value: Math.round(stage.taskDurationP50),
         recommendation: `Many small tasks (${stage.taskCount}, P50 ${Math.round(stage.taskDurationP50)}ms): scheduler overhead may dominate. Try ${fix}.`,
+        remediation: stage.shuffleReadBytes > 0 ? [decreaseConf('spark.sql.shuffle.partitions')] : [],
       };
     },
     estimate(finding, ctx)                        {
@@ -1736,6 +1841,7 @@ export const DETECTORS = [
         type: 'coldStart', stageId: null, impactBand: 'warning',
         metric: 'startupGapSeconds', value,
         recommendation: `The first stage waited ${value}s for an executor to become available: keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it doesn't scale up from zero.`,
+        remediation: dynamicAllocationOff(ctx.app) ? [] : [increaseConf('spark.dynamicAllocation.minExecutors'), increaseConf('spark.dynamicAllocation.initialExecutors')],
       };
     },
     estimate(finding)                        {
@@ -1776,13 +1882,14 @@ export const DETECTORS = [
       // CPU-time-based utilization (sparkMeasure): metric only, no threshold.
       let cpuUtilizationPct                = null;
       if (totalCores > 0) {
-        let cpuMs = 0;
-        // executorCpuTime is reported by Spark in nanoseconds.
-        for (const s of ctx.stages.values()) cpuMs += nsToMs(s.executorCpuTime ?? 0);
-        cpuUtilizationPct = Math.round((cpuMs / (appDuration * totalCores)) * 100);
+        // Null when no stage recorded CPU time (older Spark), the same rule as the CLI metrics block.
+        const cpuMs = totalExecutorCpuMs(ctx.stages.values());
+        cpuUtilizationPct = cpuMs == null ? null : Math.round((cpuMs / (appDuration * totalCores)) * 100);
       }
 
       const value = Math.round(utilization * 100);
+      const fix = dynamicAllocationFix(app, 'consider reducing cluster size or enabling dynamic allocation',
+        'dynamic allocation is already on, so consider reducing cluster size');
       return {
         type: 'utilization', stageId: null, impactBand: 'info',
         metric: 'avgUtilization', value,
@@ -1790,7 +1897,8 @@ export const DETECTORS = [
         appDurationMs: appDuration,
         totalCores,
         cpuUtilizationPct,
-        recommendation: `Average executor utilization was only ${value}%: consider reducing cluster size or enabling dynamic allocation.`,
+        recommendation: `Average executor utilization was only ${value}%: ${fix.text}.`,
+        remediation: fix.remediation,
       };
     },
     estimate(finding)                        {
@@ -1801,7 +1909,7 @@ export const DETECTORS = [
         return costOnly('measured');
       }
       const idleCoreHours = (1 - fraction) * appDurationMs * totalCores / 3.6e6;
-      return costOnly('measured', { value: idleCoreHours, unit: 'coreHours' });
+      return costOnly('measured', { value: idleCoreHours, unit: 'coreHours', idle: true });
     },
   }),
   defineAppDetector({
@@ -1838,12 +1946,15 @@ export const DETECTORS = [
         const idleRate = capacityCoreMs > 0 ? 1 - (busyCoreMs / capacityCoreMs) : 0;
         if (idleRate > thresholds.idleCoreWarn) {
           const value = Math.round(idleRate * 100);
+          const fix = dynamicAllocationFix(app, 'reduce cluster size or enable dynamic allocation',
+            'dynamic allocation is already on, so reduce cluster size');
           out.push({
             type: 'memoryUtilization', variant: 'idleCores', stageId: null,
             impactBand: 'warning', metric: 'idleCoreRate', value,
             // Raw (unrounded) rate plus sizing inputs for the impact estimator: `value` is rounded pct.
             idleRateFraction: idleRate, allocatedMB, peakExecutors, appDurationMs,
-            recommendation: `${value}% of allocated core-time ran no task: reduce cluster size or enable dynamic allocation.`,
+            recommendation: `${value}% of available core-time ran no task: ${fix.text}.`,
+            remediation: fix.remediation,
           });
         }
       }
@@ -1860,10 +1971,15 @@ export const DETECTORS = [
         }
       }
       if (peakHeapByExec.size === 0) {
+        const key = 'spark.eventLog.logStageExecutorMetrics';
+        const fix = switchFix(loggedAs(app, key, true), key, true,
+          `Per-executor memory usage requires ${key}=true: not enabled for this run.`,
+          'Per-executor memory usage is missing from this log even though executor metrics logging is on for this run.');
         out.push({
           type: 'memoryUtilization', variant: 'memoryBand', stageId: null,
           impactBand: 'info', metric: 'memoryBand', dataUnavailable: true,
-          recommendation: 'Per-executor memory usage requires spark.eventLog.logStageExecutorMetrics=true: not enabled for this run.',
+          recommendation: fix.text,
+          remediation: fix.remediation,
         });
       } else if (allocatedMB != null && allocatedMB > 0) {
         const allocatedBytes = allocatedMB * 1024 * 1024;
@@ -1877,6 +1993,7 @@ export const DETECTORS = [
               stageId: null, executorId: execId,
               impactBand: 'warning', metric: 'heapUsedRatio', value: Math.round(ratio * 100),
               recommendation: `Executor ${execId} peaked at ${Math.round(ratio * 100)}% of allocated heap: memory may be too small; raise spark.executor.memory to avoid OOM/spill.`,
+              remediation: [increaseConf('spark.executor.memory')],
             });
           } else if (ratio < thresholds.bandTooHigh) {
             out.push({
@@ -1887,6 +2004,7 @@ export const DETECTORS = [
               // unused-memory-over-time model.
               allocatedBytes, heap, appDurationMs,
               recommendation: `Executor ${execId} used only ${Math.round(ratio * 100)}% of allocated heap: memory may be over-provisioned; consider reducing spark.executor.memory for cost savings.`,
+              remediation: [decreaseConf('spark.executor.memory')],
             });
           }
         }
@@ -1907,6 +2025,7 @@ export const DETECTORS = [
             confidence: memoryWasteConfidence(wastedMBSeconds, usedMBSeconds, thresholds.wasteBufferMultiplier),
             validationRequired: `Memory-waste estimate uses allocated-vs-used memory-time and a ${thresholds.wasteBufferMultiplier}x buffer: confirm against the Spark UI before acting.`,
             recommendation: `Allocated executor memory sat largely idle over the run (~${value.toLocaleString('en-US')} MB-seconds wasted): review spark.executor.memory and executor count.`,
+            remediation: [decreaseConf('spark.executor.memory')],
           });
         }
       }
@@ -1994,7 +2113,7 @@ export const DETECTORS = [
           }
         }
       }
-      if (persistedRddCount > 0 && !anyStorageEvidence) out.push(storageUnobservedFinding(persistedRddCount));
+      if (persistedRddCount > 0 && !anyStorageEvidence) out.push(storageUnobservedFinding(persistedRddCount, ctx.app));
       return out;
     },
     estimate(finding)                        {
@@ -2038,8 +2157,8 @@ export const DETECTORS = [
         // Raw count behind the ratio, for the impact estimator. Non-null whenever totalTasks is.
         nonLocalTaskCount: nonLocalTasks ,
         confidence: coreLocalityConfidence(ratio , totalTasks, thresholds),
-        validationRequired: `This finding is gated by ${shareLabel(thresholds.warnRatio)}/${shareLabel(thresholds.critRatio)} non-local-ratio thresholds (and a ${thresholds.minTasks}-task minimum), our own noise floor for this metric.`,
-        recommendation: `${value}% of tasks (${nonLocalTasks }) ran without process- or node-local data placement: check spark.locality.wait settings and executor/data colocation.`,
+        validationRequired: `Flagged when at least ${shareLabel(thresholds.warnRatio)} of tasks run non-local (critical at ${shareLabel(thresholds.critRatio)}), on runs of ${thresholds.minTasks}+ tasks.`,
+        recommendation: `${value}% of tasks (${nonLocalTasks }) ran without process- or node-local data placement: check executor/data colocation.`,
       };
     },
     estimate(finding)                        {
@@ -2083,7 +2202,12 @@ export const DETECTORS = [
         // Raw count behind the percentage, for the impact estimator's startup-overhead figure.
         shortLivedExecutorCount: shortLivedCount,
         confidence: autoscalingChurnConfidence(shortLivedPct, thresholds.warningPct, thresholds.criticalPct),
-        recommendation: `${pct}% of executors ran for under 2 minutes before being removed. This looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
+        recommendation: `${pct}% of executors ran for under 2 minutes before being removed: this looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
+        remediation: [
+          increaseConf('spark.dynamicAllocation.executorIdleTimeout'),
+          decreaseConf('spark.dynamicAllocation.minExecutors'),
+          increaseConf('spark.dynamicAllocation.maxExecutors'),
+        ],
       };
     },
     estimate(finding)                        {
@@ -2221,8 +2345,8 @@ export const DETECTORS = [
         const relationDisplay = rids.map(relationDisplayName).join(` ${connector} `);
         const value = finalExecutionIds.length;
         const recommendation = totalReadBytes >= 128 * MB
-          ? `${verb[0].toUpperCase()}${verb.slice(1)} result read by ${value} queries (~${formatBytes(totalReadBytes)}). Cache/persist the ${verb} DataFrame so it is computed once.`
-          : `${verb[0].toUpperCase()}${verb.slice(1)} result read by ${value} queries. Cache the ${verb} DataFrame, or reconsider whether it needs to be recomputed each time.`;
+          ? `${verb[0].toUpperCase()}${verb.slice(1)} result read by ${value} queries (~${formatBytes(totalReadBytes)}): cache/persist the ${verb} DataFrame so it is computed once.`
+          : `${verb[0].toUpperCase()}${verb.slice(1)} result read by ${value} queries: cache the ${verb} DataFrame, or reconsider whether it needs to be recomputed each time.`;
 
         out.push({
           type: 'cachingOpportunity', variant: 'composite', stageId: null, impactBand: 'info',
@@ -2252,8 +2376,8 @@ export const DETECTORS = [
         const value = residualExecutionIds.length;
         const totalReadBytes = residualExecutionIds.reduce((sum, id) => sum + (agg.executionBytes.get(id) ?? 0), 0);
         const recommendation = totalReadBytes >= 128 * MB
-          ? `Read by ${value} queries (~${formatBytes(totalReadBytes)}). Cache/persist the shared DataFrame so it is scanned once.`
-          : `Read by ${value} queries. Cache the shared DataFrame, or broadcast it if it is a small join lookup.`;
+          ? `Read by ${value} queries (~${formatBytes(totalReadBytes)}): cache/persist the shared DataFrame so it is scanned once.`
+          : `Read by ${value} queries: cache the shared DataFrame, or broadcast it if it is a small join lookup.`;
         out.push({
           type: 'cachingOpportunity', stageId: null, impactBand: 'info',
           metric: 'executionReuse', value,
@@ -2328,6 +2452,7 @@ export const DETECTORS = [
           type: 'configAudit', property: 'spark.shuffle.service.enabled',
           impactBand: 'warning', metric: 'config', valueText: 'false',
           recommendation: 'Dynamic allocation is on but the external shuffle service is off: set spark.shuffle.service.enabled=true so shuffle data survives executor removal.',
+          remediation: setConfUnlessLogged(target.app, 'spark.shuffle.service.enabled', true),
         };
       }
       return null;
@@ -2347,7 +2472,8 @@ export const DETECTORS = [
         return {
           type: 'configAudit', property: 'spark.dynamicAllocation.minExecutors',
           impactBand: 'critical', metric: 'config', valueText: `${minN} > ${maxN}`,
-          recommendation: `Autoscaling bounds are inverted: spark.dynamicAllocation.minExecutors (${minN}) exceeds maxExecutors (${maxN}). Set min ≤ max.`,
+          recommendation: `spark.dynamicAllocation.minExecutors (${minN}) exceeds maxExecutors (${maxN}): set min ≤ max.`,
+          remediation: [decreaseConf('spark.dynamicAllocation.minExecutors', maxN)],
         };
       }
       if (maxN == null) {
@@ -2355,6 +2481,7 @@ export const DETECTORS = [
           type: 'configAudit', property: 'spark.dynamicAllocation.maxExecutors',
           impactBand: 'info', metric: 'config', valueText: '(unset)',
           recommendation: 'Dynamic allocation is on with no upper bound: set spark.dynamicAllocation.maxExecutors to cap cluster growth.',
+          remediation: [setConf('spark.dynamicAllocation.maxExecutors')],
         };
       }
       return null;
@@ -2375,6 +2502,7 @@ export const DETECTORS = [
         type: 'configAudit', property: 'spark.serializer',
         impactBand: 'info', metric: 'config', valueText: ser ?? '(default JavaSerializer)',
         recommendation: `Current serializer is ${ser ?? 'the default JavaSerializer'}: consider spark.serializer=org.apache.spark.serializer.KryoSerializer for faster, smaller buffers.`,
+        remediation: setConfUnlessLogged(app, 'spark.serializer', 'org.apache.spark.serializer.KryoSerializer'),
       };
     },
     estimate: noWasteModel,
@@ -2394,6 +2522,7 @@ export const DETECTORS = [
         type: 'configAudit', property: 'spark.executor.memoryOverhead',
         impactBand: 'info', metric: 'config', valueText: `${ovMB} MiB`,
         recommendation: `Executor memoryOverhead (${ovMB} MiB) is below Spark's default floor of ${floor} MiB (max of 384 MiB or 10% of executor memory): raise it to avoid off-heap OOM-kills.`,
+        remediation: [increaseConf('spark.executor.memoryOverhead', `${floor}m`)],
       };
     },
     estimate: noWasteModel,
@@ -2431,8 +2560,8 @@ export const DETECTORS = [
         const stageShares = stageOperatorShares(nodes, operatorsByStage);
         // resolvePlanTree always sets id; safe downstream of it.
         const planNodeIds = nodes.map((n) => n.id ).filter(Boolean);
-        const touching = g.sampleRelation ? ` (touching ${g.sampleRelation})` : '';
-        const differing = occurrencesIdentical ? '' : ' Their filters, columns or scanned tables differ, so the repeats may compute different data.';
+        const detail = duplicateSubtreeDetail({ ...g, value: g.occurrences });
+        const differing = occurrencesIdentical ? '' : ` ${DUPLICATE_SUBTREE_DIFFERING_NOTE}`;
         return {
           type: 'duplicatePlanSubtree', executionId: sqlExec.id, stageIds, planNodeIds,
           stageShares, occurrencesIdentical,
@@ -2444,10 +2573,10 @@ export const DETECTORS = [
           rootName: g.rootName, subtreeSize: g.subtreeSize, sampleRelation: g.sampleRelation,
           groupIndex: g.groupIndex,
           confidence: occurrencesIdentical ? duplicateSubtreeConfidence(g.subtreeSize, g.occurrences, thresholds) : 'low',
-          validationRequired: 'Duplicate-subtree matching compares operator names and metric names only, not literal values or expr IDs: confirm the repeated work is real in the Spark SQL plan tab before acting.',
+          validationRequired: 'Matching compares operator and metric names only, not literals or expression IDs: confirm the repeat in the Spark UI SQL tab before acting.',
           recommendation: (g.isExchangeRoot
-            ? `A ${g.subtreeSize}-node subtree rooted at ${pathBasename(g.rootName)} repeats ${g.occurrences}x in this plan${touching}: this looks like a possible missed exchange reuse; check whether the same shuffle could be computed once and reused.`
-            : `A ${g.subtreeSize}-node subtree rooted at ${pathBasename(g.rootName)} repeats ${g.occurrences}x in this plan${touching}: consider caching/persisting the shared computation or check for a duplicated query branch.`) + differing,
+            ? `${detail}: this looks like a possible missed exchange reuse; check whether the same shuffle could be computed once and reused.`
+            : `${detail}: consider caching/persisting the shared computation or check for a duplicated query branch.`) + differing,
         };
       }).filter((f)               => f !== null);
       return findings.length > 0 ? findings : null;
@@ -2583,6 +2712,7 @@ export const DETECTORS = [
                 impactBand: 'info', metric: 'smallerSideBytes', value: smaller,
                 largerSideBytes: larger,
                 recommendation: `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is well under the broadcast threshold relative to the larger side (${formatBytes(larger)}): this could have been a broadcast join. Consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.`,
+                remediation: [increaseConf('spark.sql.autoBroadcastJoinThreshold')],
               });
             }
           }
@@ -2594,13 +2724,17 @@ export const DETECTORS = [
             // (node.stageIds always empty in real data); its child carries the executor-side
             // metrics, so only the child unions in.
             const child = (node.children ?? [])[0];
+            const autoBroadcastOff = ctx.app?.config?.['spark.sql.autoBroadcastJoinThreshold']?.trim() === '-1';
             out.push({
               type: 'overBroadcast', executionId: sqlExec.id,
               stageIds: unionStageIds(child ? [child] : [], fallbackStageIds),
               // resolvePlanTree always sets id; safe downstream of it.
               planNodeIds: [node.id ].filter(Boolean),
               impactBand: 'warning', metric: 'broadcastBytes', value: m.value,
-              recommendation: `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
+              recommendation: autoBroadcastOff
+                ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: automatic broadcast is already disabled, so remove the broadcast() hint that forced it.`
+                : `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
+              remediation: autoBroadcastOff ? [] : [decreaseConf('spark.sql.autoBroadcastJoinThreshold')],
             });
           }
         }