sparkforensics-server 0.5.1 → 0.6.0

package/vendor-core/detectors.js

@@ -1,6 +1,8 @@
-import { pathBasename, formatBytes, IMPACT_BAND_ORDER } from './format-utils.js';
+import { pathBasename, formatBytes, IMPACT_BAND_ORDER, MS_PER_CORE_HOUR } from './format-utils.js';
+import { medianOfSorted } from './median.js';
 import { shareLabel } from './finding-presentation.js';
 import { scanRelationId } from './plan-summary.js';
+import { allocatedCoreMs, computeAllocation } from './allocation.js';
 import { computePeakConcurrentCores, computePeakConcurrentExecutorCount } from './core-count.js';
 import { walkPlanTree } from './plan-tree-walk.js';
 import { computeCoreLocalityRatio } from './core-locality-ratio.js';
@@ -18,10 +20,10 @@ import { totalExecutorCpuMs } from './run-totals.js';
 import { DUPLICATE_SUBTREE_DIFFERING_NOTE, duplicateSubtreeDetail, SLOW_HOST_DIMENSION_LABEL } from './finding-generic-recommendation.js';
 import { stageIdsForSqlExec } from './sql-stages.js';
 import { cyrb53 } from './string-hash.js';
-import { decreaseConf, increaseConf, setConf } from './remediation.js';
+import { codeFix, decreaseConf, increaseConf, setConf } from './remediation.js';
 import { MAX_FAILURE_GROUPS, describeTaskFailure,                       } from './task-failure.js';
                                                                                                
-                                                                                                                      
+                                                                                                                                                                                     
 
 const MB = 1024 * 1024;
 const GB = 1024 * MB;
@@ -543,7 +545,7 @@ function maxMedianRatio(
 )                                                       {
   if (samples.length < 3) return null;
   const vals = samples.map(s => s.value).sort((a, b) => a - b);
-  const median = vals[Math.floor(vals.length / 2)];
+  const median = medianOfSorted(vals);
   if (median <= 0) return null;
   const top = samples.reduce((a, b) => (b.value > a.value ? b : a));
   return { key: top.key, ratio: top.value / median, value: top.value };
@@ -655,14 +657,36 @@ function tailClaimFloorMs(claim           , stageId        , ctx             )
   return tailClaimImpact(claim, stageId, ctx.impact).wallClock?.high ?? claim.wasteMs;
 }
 
-// A setting is only a fix when the run's logged conf doesn't already have it: a run that set it
-// needs another remedy. Only explicitly logged properties count; Spark's unlogged version
-// defaults are not modeled. Booleans compare case-insensitively, as Spark parses them.
+// Spark's default for the properties a detector's fix reads or suggests, for a run that did not log
+// them. Undefined when the property is not modeled or its default depends on a version the log did
+// not record. A property is modeled only where its default has made a suggestion a no-op; every
+// other property a detector suggests defaults off, or to a value other than the suggested one.
+function versionDefault(app                    , key        )                     {
+  if (key === 'spark.sql.shuffle.partitions') return '200';
+  if (key === 'spark.sql.autoBroadcastJoinThreshold') return String(10 * MB);
+  const version = /^(\d+)\.(\d+)/.exec(app?.sparkVersion ?? '');
+  if (version == null) return undefined;
+  const [major, minor] = [Number(version[1]), Number(version[2])];
+  if (key === 'spark.sql.adaptive.enabled') return major > 3 || (major === 3 && minor >= 2) ? 'true' : 'false';
+  if (key === 'spark.sql.adaptive.skewJoin.enabled' || key === 'spark.sql.adaptive.coalescePartitions.enabled') {
+    return major >= 3 ? 'true' : undefined;
+  }
+  return undefined;
+}
+
+// The run's effective value of a property: the logged one, else versionDefault's.
+function effectiveConf(app                    , key        )                     {
+  return (app?.config?.[key] ?? versionDefault(app, key))?.trim();
+}
+
+// A setting is only a fix when the run's effective conf doesn't already have it: a run that set it,
+// or whose Spark version defaults to it, needs another remedy. Booleans compare case-insensitively,
+// as Spark parses them.
 function loggedAs(app                    , key        , suggested                  )          {
-  const logged = app?.config?.[key]?.trim();
+  const effective = effectiveConf(app, key);
   return typeof suggested === 'boolean'
-    ? logged?.toLowerCase() === String(suggested)
-    : logged === suggested;
+    ? effective?.toLowerCase() === String(suggested)
+    : effective === suggested;
 }
 
 function setConfUnlessLogged(app                    , key        , suggested                  )                {
@@ -675,18 +699,69 @@ function switchFix(on         , key        , suggested                  , recomm
   return on ? { text: alreadyOn, remediation: [] } : { text: recommend, remediation: [setConf(key, suggested)] };
 }
 
+const SKEW_KEY_REMEDY = 'salt the key or repartition on a better key';
+
+// What a stage reads, by the dominant side: a stage that reads a shuffle and mostly input files is
+// an input stage for any fix that sizes a shuffle's reduce side.
+function stageReads(stage                                                        )             {
+  if (stage.shuffleReadBytes > 0 && stage.shuffleReadBytes >= stage.inputBytes) return 'shuffle';
+  return stage.inputBytes > 0 ? 'input' : 'other';
+}
+
+// Whether the run's Spark predates 3.0, which has no AQE skew-join handling to suggest.
+function predatesAqeSkewJoin(app                    )          {
+  const major = /^(\d+)\./.exec(app?.sparkVersion ?? '');
+  return major != null && Number(major[1]) < 3;
+}
+
+// What a skewed stage reads, which decides whether AQE skew-join handling can act on it. It splits
+// skewed partitions on the shuffle-read side of a sort-merge or shuffled-hash join, so only a stage
+// that reads a shuffle, in a SQL execution whose plan has such a join, is 'shuffleJoin'. A plan does
+// not say which stage runs which join (a node's stageIds are empty in real logs), so the join is
+// matched per execution. 'inputScan' reads mostly files (its tasks differ in input size);
+// 'other' is any stage the plan cannot tie to a join (an aggregation's shuffle, no plan, no input),
+// and any stage of a run on Spark before 3.0 (skewFix).
+function skewOrigin(stage               , ctx             , shuffleEvidence         )                                        {
+  if (shuffleEvidence || (stage.shuffleReadBytes > 0 && stage.shuffleReadBytes >= stage.inputBytes)) {
+    let hasJoin = false;
+    const plan = stage.sqlExecutionId != null ? ctx.sql.get(stage.sqlExecutionId)?.planTree : null;
+    walkPlanTree(plan, (node) => { if (SKEW_JOIN_NODES.has(node.name)) hasJoin = true; });
+    return hasJoin ? 'shuffleJoin' : 'other';
+  }
+  return stage.inputBytes > 0 ? 'inputScan' : 'other';
+}
+const SKEW_JOIN_NODES = new Set(['SortMergeJoin', 'ShuffledHashJoin']);
+
+// The skew finding's fix for the stage's origin. Join-driven skew gets AQE skew-join handling,
+// unless the run's effective conf already has it; uneven input gets the file-size remedy.
+function skewFix(stage               , ctx             , shuffleEvidence = false)                                                                   {
+  const origin = skewOrigin(stage, ctx, shuffleEvidence);
+  if (origin === 'inputScan') {
+    return {
+      origin,
+      text: 'the stage reads uneven input files: compact small files or split large ones (lower spark.sql.files.maxPartitionBytes)',
+      remediation: [decreaseConf('spark.sql.files.maxPartitionBytes')],
+    };
+  }
+  if (origin === 'other') return { origin, text: SKEW_KEY_REMEDY, remediation: [codeFix(SKEW_KEY_REMEDY)] };
+  // Before Spark 3.0 there is no AQE skew-join handling, so the stage gets the generic advice.
+  if (predatesAqeSkewJoin(ctx.app)) return { origin: 'other', text: SKEW_KEY_REMEDY, remediation: [codeFix(SKEW_KEY_REMEDY)] };
+  return { origin, ...skewJoinFix(ctx.app) };
+}
+
 function skewJoinFix(app                    )                                               {
   const key = 'spark.sql.adaptive.skewJoin.enabled';
-  const remedy = 'salt the key or repartition on a better key';
   if (loggedAs(app, 'spark.sql.adaptive.enabled', false)) {
     return {
-      text: `AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise ${remedy}`,
+      text: `AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise ${SKEW_KEY_REMEDY}`,
       remediation: [setConf('spark.sql.adaptive.enabled', true), ...setConfUnlessLogged(app, key, true)],
     };
   }
-  return switchFix(loggedAs(app, key, true), key, true,
-    `for join-driven skew, enable AQE skew-join handling (${key}); otherwise ${remedy}`,
-    `AQE skew-join handling is already on, so ${remedy}`);
+  const fix = switchFix(loggedAs(app, key, true), key, true,
+    `for join-driven skew, enable AQE skew-join handling (${key}); otherwise ${SKEW_KEY_REMEDY}`,
+    `AQE skew-join handling is already on, so ${SKEW_KEY_REMEDY}`);
+  // With the switch already on, only the key is left to fix: a change to the job, not a property.
+  return fix.remediation.length === 0 ? { ...fix, remediation: [codeFix(SKEW_KEY_REMEDY)] } : fix;
 }
 
 // The resources flag is read from the same property as the logged conf.
@@ -695,31 +770,92 @@ function dynamicAllocationFix(app             , recommend        , alreadyOn
   return switchFix(app.resources?.dynamicAllocationEnabled === true || loggedAs(app, key, true), key, true, recommend, alreadyOn);
 }
 
-// The run's logged spark.sql.shuffle.partitions as a count, or null when unlogged or not a count.
-function loggedShufflePartitions(app                    )                {
-  const logged = app?.config?.['spark.sql.shuffle.partitions']?.trim();
-  return logged != null && /^\d+$/.test(logged) ? Number(logged) : null;
+// Idle capacity is cured by a smaller cluster, which the run's dynamic allocation decides how to
+// size. With it on: the cap on how many executors it scales to, and the floor it holds when that
+// is logged above 0 (both lowered together). With it off or unset: the alternatives of a fixed
+// executor count or switching it on, which dynamicAllocationFix already names, so the
+// recommendation reads "either ... or" and a consumer applies one entry. Not executorIdleTimeout:
+// autoscalingChurn recommends raising it, so lowering it here would contradict that finding.
+function idleCapacityFix(app             , recommend        , alreadyOnLead        )                                               {
+  const fix = dynamicAllocationFix(app, recommend, alreadyOnLead);
+  // dynamicAllocationFix's remediation is empty exactly when dynamic allocation is already on.
+  if (fix.remediation.length > 0) return { ...fix, remediation: [...fix.remediation, decreaseConf('spark.executor.instances')] };
+  const holdsFloor = Number.parseInt(app.config?.['spark.dynamicAllocation.minExecutors'] ?? '', 10) > 0;
+  return holdsFloor
+    ? { text: `${alreadyOnLead} by lowering spark.dynamicAllocation.maxExecutors and spark.dynamicAllocation.minExecutors`,
+        remediation: [decreaseConf('spark.dynamicAllocation.maxExecutors'), decreaseConf('spark.dynamicAllocation.minExecutors')] }
+    : { text: `${alreadyOnLead} by lowering spark.dynamicAllocation.maxExecutors`, remediation: [decreaseConf('spark.dynamicAllocation.maxExecutors')] };
+}
+
+// The run's effective spark.sql.shuffle.partitions (logged, else Spark's 200) as a count, or null
+// when the logged value is not a count.
+function effectiveShufflePartitions(app                    )                {
+  const value = effectiveConf(app, 'spark.sql.shuffle.partitions');
+  return value != null && /^\d+$/.test(value) ? Number(value) : null;
+}
+
+// Whether partition-count advice fits a shuffle-reading stage, from the effective conf and the
+// stage's own task sizes. The partition count that brings each shuffle partition down to the ideal
+// size is per stage; the property is job-wide. 'sufficient': the stage's tasks are already at or
+// under the ideal size. 'aqeCoalesced': AQE merged the property's partitions into fewer tasks, so
+// the advisory partition size is the lever. 'ownPartitioning': the property already gives at least
+// the count needed, so the stage's own repartition(n) or RDD parallelism limits it. Otherwise the
+// property limits the stage: 'raise'.
+function shufflePartitionCase(stage               , app                    )                                                                          {
+  const count = effectiveShufflePartitions(app);
+  const needed = Math.ceil(stage.shuffleReadBytes / IDEAL_BYTES_PER_PARTITION_TASK);
+  const tasks = Math.max(1, stage.taskCount);
+  let partitions                    = 'raise';
+  if (stage.shuffleReadBytes / tasks <= IDEAL_BYTES_PER_PARTITION_TASK) partitions = 'sufficient';
+  else if (count != null && stage.taskCount < count
+    && loggedAs(app, 'spark.sql.adaptive.enabled', true) && loggedAs(app, 'spark.sql.adaptive.coalescePartitions.enabled', true)) partitions = 'aqeCoalesced';
+  else if (count != null && count >= needed) partitions = 'ownPartitioning';
+  return { partitions, count, needed };
 }
 
-// lowShuffleParallelism's fix. The partition count that brings each shuffle partition down to the
-// ideal size, as the estimate models it, is per stage; the property is job-wide. A logged value at
-// or above it means the property is not what limits that stage (a repartition(n) or an RDD
-// shuffle is), so the text points at the stage's own partitioning and no property is suggested.
-// Unlogged, the count is not a safe value: null.
-function lowShuffleParallelismFix(app                    , needed        )                                               {
-  const logged = loggedShufflePartitions(app);
-  if (logged != null && logged >= needed) {
+const ADVISORY_PARTITION_SIZE_KEY = 'spark.sql.adaptive.advisoryPartitionSizeInBytes';
+
+// lowShuffleParallelism's fix. Unlike the shuffle finding it fires on stages whose tasks are larger
+// than the ideal size, so its cases are 'raise', 'aqeCoalesced' and 'ownPartitioning'.
+function lowShuffleParallelismFix(stage               , app                    )                                                                              {
+  const { partitions, count, needed } = shufflePartitionCase(stage, app);
+  if (partitions === 'aqeCoalesced') {
+    return {
+      partitions,
+      text: `AQE coalesced the shuffle into ${stage.taskCount} tasks (from ${count} configured partitions): lower ${ADVISORY_PARTITION_SIZE_KEY} so each partition is smaller`,
+      remediation: [decreaseConf(ADVISORY_PARTITION_SIZE_KEY)],
+    };
+  }
+  if (partitions === 'ownPartitioning') {
     return {
-      text: `spark.sql.shuffle.partitions is already ${logged}, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller`,
+      partitions,
+      text: `spark.sql.shuffle.partitions is already ${count}, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller`,
       remediation: [],
     };
   }
   return {
+    partitions,
     text: 'raise spark.sql.shuffle.partitions so each partition is smaller',
-    remediation: [increaseConf('spark.sql.shuffle.partitions', logged == null ? null : needed)],
+    remediation: [increaseConf('spark.sql.shuffle.partitions', count == null ? null : needed)],
   };
 }
 
+// A Spark byte-size property as bytes (a plain number, or with a k/m/g/t suffix, optionally 'b'), or
+// null when it is not one. Negative numbers pass through: -1 turns auto-broadcast off.
+function parseSparkBytes(value                    )                {
+  const m = /^(-?\d+(?:\.\d+)?)\s*([kmgt]?)b?$/i.exec(value?.trim() ?? '');
+  if (m == null) return null;
+  return m[2] === '' ? Number(m[1]) : Number(m[1]) * 1024 ** ('kmgt'.indexOf(m[2].toLowerCase()) + 1);
+}
+
+// The run's effective spark.sql.autoBroadcastJoinThreshold in bytes (logged, else Spark's 10 MiB);
+// negative when auto-broadcast is disabled, null when the logged value is not a size.
+function effectiveBroadcastThreshold(app                    )                {
+  const raw = effectiveConf(app, 'spark.sql.autoBroadcastJoinThreshold');
+  if (raw != null && /^-\d+$/.test(raw)) return -1;
+  return parseSparkBytes(raw);
+}
+
 // No dynamic-allocation property has an effect on a run whose logged conf turns it off.
 function dynamicAllocationOff(app                    )          {
   return app?.resources?.dynamicAllocationEnabled === false || app?.config?.['spark.dynamicAllocation.enabled']?.trim().toLowerCase() === 'false';
@@ -1061,9 +1197,9 @@ export const DETECTORS = [
       const floorWasteMs = tailClaimFloorMs(skewTailClaim(stage, metric === 'P95/median'), stage.id, ctx);
       if (!meetsRuntimeFloor(floorWasteMs, appDurationMs(ctx.app), thresholds.floorPctWarn)) return null;
       const value = Math.round(ratio * 10) / 10;
-      const fix = skewJoinFix(ctx.app);
+      const fix = skewFix(stage, ctx);
       return {
-        type: 'skew', stageId: stage.id,
+        type: 'skew', stageId: stage.id, origin: fix.origin,
         impactBand: 'warning',
         metric, value,
         confidence: skewConfidence(ratio, thresholds.ratioWarn),
@@ -1188,12 +1324,21 @@ export const DETECTORS = [
       const bytes = stage.shuffleReadBytes;
       if (bytes <= thresholds.minBytes) return null;
       if (stageBelowRuntimeFloor(stage, ctx, thresholds.stageFloorPct)) return null;
+      const { partitions, count } = shufflePartitionCase(stage, ctx.app);
+      const perTask = formatBytes(bytes / Math.max(1, stage.taskCount));
+      const fix = partitions === 'sufficient'
+        ? { text: `its ${stage.taskCount} tasks already read about ${perTask} each, so more partitions will not help: consider a broadcast join for the smaller side`, remediation: [] }
+        : partitions === 'aqeCoalesced'
+          ? { text: `AQE coalesced it into ${stage.taskCount} tasks (from ${count} configured partitions): consider lowering ${ADVISORY_PARTITION_SIZE_KEY} or adding a broadcast join`, remediation: [decreaseConf(ADVISORY_PARTITION_SIZE_KEY)] }
+          : partitions === 'ownPartitioning'
+            ? { text: `spark.sql.shuffle.partitions is already ${count}, so consider raising this stage's own partition count (its repartition(n) or RDD parallelism) or adding a broadcast join`, remediation: [] }
+            : { text: 'consider increasing spark.sql.shuffle.partitions or adding a broadcast join', remediation: [increaseConf('spark.sql.shuffle.partitions')] };
       return {
         type: 'shuffle', stageId: stage.id,
-        impactBand: 'info',
+        impactBand: 'info', partitions,
         metric: 'shuffleReadBytes', value: bytes,
-        recommendation: `${formatBytes(bytes)} shuffled in this stage: consider increasing spark.sql.shuffle.partitions or adding a broadcast join.`,
-        remediation: [increaseConf('spark.sql.shuffle.partitions')],
+        recommendation: `${formatBytes(bytes)} shuffled in this stage: ${fix.text}.`,
+        remediation: fix.remediation,
       };
     },
     estimate(finding, ctx)                        {
@@ -1225,19 +1370,19 @@ export const DETECTORS = [
         const ratioText = p50 > 0
           ? `${Math.round(max / p50 * 10) / 10}× the median (${formatBytes(p50)})`
           : 'far larger than the median, which is effectively empty';
-        const fix = skewJoinFix(ctx.app);
+        const fix = skewFix(stage, ctx, true);
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
-          rule: 'shufflePartitionSkew', metric: 'shuffleReadMax', value: max,
+          rule: 'shufflePartitionSkew', origin: fix.origin, metric: 'shuffleReadMax', value: max,
           recommendation: `The largest shuffle partition (${formatBytes(max)}) is ${ratioText}: ${fix.text}.`,
           remediation: fix.remediation,
         });
       }
       if (total >= thresholds.lowParTotalBytes && taskCount <= thresholds.lowParMaxTasks) {
-        const fix = lowShuffleParallelismFix(ctx.app, Math.ceil(total / IDEAL_BYTES_PER_PARTITION_TASK));
+        const fix = lowShuffleParallelismFix(stage, ctx.app);
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
-          rule: 'lowShuffleParallelism', metric: 'taskCount', value: taskCount,
+          rule: 'lowShuffleParallelism', partitions: fix.partitions, metric: 'taskCount', value: taskCount,
           recommendation: `${Math.round(total / GB * 10) / 10} GB of shuffle spread over only ${taskCount} tasks: ${fix.text}.`,
           remediation: fix.remediation,
         });
@@ -1260,6 +1405,7 @@ export const DETECTORS = [
       const stage = ctx.stages.get(finding.stageId);
       if (!stage) return null;
       let wasteMs = 0;
+      let longestTaskAfterFixMs = 0;
       if (finding.rule === 'maxPartitionTooBig') {
         wasteMs = ((stage.shuffleReadMax ?? 0) / SHUFFLE_THROUGHPUT_BPS) * 1000;
       } else if (finding.rule === 'shufflePartitionSkew') {
@@ -1275,11 +1421,21 @@ export const DETECTORS = [
           // work, not the scheduling cost of tasks you'd add (adding tasks incurs overhead, recovers
           // nothing). Model the achievable duration at target parallelism by scaling down proportionally.
           wasteMs = stageDurationMs * (1 - taskCount / targetTaskCount);
+          // The fix splits the long tasks, so the longest task is the quantity it shortens: the
+          // occupancy clip must not floor the claim at it (TAIL_CLAIM). An even split leaves the
+          // longest task at taskCount / targetTaskCount of today's: a modeled figure.
+          const longestTaskMs = stage.taskDurationMax ?? 0;
+          longestTaskAfterFixMs = longestTaskMs * taskCount / targetTaskCount;
+          // Splitting only shortens tasks: the stage still takes the time outside its longest task
+          // plus the longest task the split leaves, so the claim is at most the longest task's own
+          // reduction. Without a task-duration max (older snapshots) the stage-wide scaling stands.
+          if (longestTaskMs > 0) wasteMs = Math.min(wasteMs, longestTaskMs - longestTaskAfterFixMs);
         }
       } else {
         return null;
       }
-      return singleStageImpact(wasteMs, finding.stageId, ctx, 'modeled', { value: wasteMs, unit: 'ms' });
+      return singleStageImpact(wasteMs, finding.stageId, ctx, 'modeled', { value: wasteMs, unit: 'ms' },
+        finding.rule === 'lowShuffleParallelism' ? { ...TAIL_CLAIM, longestTaskAfterFixMs } : undefined);
     },
   }),
   defineStageDetector({
@@ -1296,8 +1452,11 @@ export const DETECTORS = [
       const classified = cls === 'skew' || cls === 'volume';
       const mag = computeSpillMagnitude(stage, thresholds);
       const impactBand = 'warning';
+      // shuffle.partitions sizes a shuffle's reduce side: a stage that reads no shuffle gets memory only.
+      const reads = stageReads(stage);
+      const partitionAdvice = reads === 'shuffle';
       return {
-        type: 'spill', stageId: stage.id, impactBand,
+        type: 'spill', stageId: stage.id, impactBand, reads,
         spillMagnitude: mag?.magnitude,
         metric: 'memoryBytesSpilled', value: stage.memoryBytesSpilled,
         confidence: classified ? 'medium' : 'low',
@@ -1306,8 +1465,10 @@ export const DETECTORS = [
           : 'Spill cause could not be classified: inspect per-task spill metrics in the Spark UI before acting.',
         recommendation: cls === 'skew'
           ? `${formatBytes(stage.memoryBytesSpilled)} spilled, skew-driven: fix task skew first; adding memory will not help.`
-          : `${formatBytes(stage.memoryBytesSpilled)} spilled: raise spark.sql.shuffle.partitions or increase executor memory.`,
-        remediation: cls === 'skew' ? [] : [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.executor.memory')],
+          : partitionAdvice
+            ? `${formatBytes(stage.memoryBytesSpilled)} spilled: raise spark.sql.shuffle.partitions or increase executor memory.`
+            : `${formatBytes(stage.memoryBytesSpilled)} spilled: this stage reads no shuffle, so increase executor memory or process less data per task.`,
+        remediation: cls === 'skew' ? [] : [...(partitionAdvice ? [increaseConf('spark.sql.shuffle.partitions')] : []), increaseConf('spark.executor.memory')],
       };
     },
     estimate(finding, ctx)                        {
@@ -1419,7 +1580,7 @@ export const DETECTORS = [
       if (hosts.length >= thresholds.minHosts) {
         const means = hosts.map(h => ({ host: h.host, taskCount: h.taskCount, mean: h.totalDuration / h.taskCount }));
         const sorted = [...means].map(h => h.mean).sort((a, b) => a - b);
-        const overallMedian = sorted[Math.floor(sorted.length / 2)];
+        const overallMedian = medianOfSorted(sorted);
         if (overallMedian > 0) {
           for (const h of means) {
             const ratio = h.mean / overallMedian;
@@ -1532,11 +1693,21 @@ export const DETECTORS = [
       const impactBand = durationMinutes >= t.infoMin ? 'info' : null;
       if (!impactBand) return null;
       const value = Math.round(durationMinutes * 10) / 10;
+      // shuffle.partitions and default.parallelism size a shuffle's reduce side: a stage that reads
+      // no shuffle gets the input-partitioning remedy instead.
+      const reads = stageReads(stage);
+      const recommendation = reads === 'shuffle'
+        ? `This stage ran ${value} minutes with no more specific cause flagged: often a partition-count problem, raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.`
+        : reads === 'input'
+          ? `This stage ran ${value} minutes with no more specific cause flagged and reads mostly input files: often too few or too uneven input partitions, so check input file sizes and lower spark.sql.files.maxPartitionBytes, or look for a large per-task data volume driving heavy spill.`
+          : `This stage ran ${value} minutes with no more specific cause flagged and reads neither shuffle nor input files: check what it computes and for a large per-task data volume driving heavy spill.`;
       return {
         type: 'stageSlowness', stageId: stage.id, impactBand,
-        metric: 'stageDurationMinutes', value,
-        recommendation: `This stage ran ${value} minutes with no more specific cause flagged: often a partition-count problem, raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.`,
-        remediation: [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.default.parallelism')],
+        metric: 'stageDurationMinutes', value, reads,
+        recommendation,
+        remediation: reads === 'shuffle'
+          ? [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.default.parallelism')]
+          : reads === 'input' ? [decreaseConf('spark.sql.files.maxPartitionBytes')] : [],
       };
     },
     estimate(finding, ctx)                        {
@@ -1664,8 +1835,10 @@ export const DETECTORS = [
       const detail = useSpeculativeMetric
         ? `${value} speculative attempt${value === 1 ? '' : 's'} discarded`
         : `${value}% of tasks straggled`;
+      // The skew advice fits only a stage that reads a shuffle feeding a join (skewFix).
+      const fix = skewFix(stage, ctx);
       return {
-        type: 'straggler', stageId: stage.id, impactBand,
+        type: 'straggler', stageId: stage.id, impactBand, origin: fix.origin,
         metric: useSpeculativeMetric ? 'speculativeTasks' : 'stragglerShare',
         value,
         unit: useSpeculativeMetric ? 'count' : 'pct',
@@ -1675,7 +1848,8 @@ export const DETECTORS = [
           ? stragglerConfidence(speculativeShare, thresholds.warnPct, thresholds.critPct)
           : stragglerConfidence(stragglerShare, thresholds.shareWarn, thresholds.critPct),
         validationRequired: `Warning needs at least ${shareLabel(thresholds.floorPctWarn)} of run time at stake, critical ${shareLabel(thresholds.floorPctCrit)}.`,
-        recommendation: `${detail}: rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that's a candidate for AQE's skew-join handling.`,
+        recommendation: `${detail}: rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if uneven data is the cause, ${fix.text}.`,
+        remediation: fix.remediation,
       };
     },
     estimate(finding, ctx)                        {
@@ -1755,14 +1929,15 @@ export const DETECTORS = [
       if (stageBelowRuntimeFloor(stage, ctx, thresholds.stageFloorPct)) return null;
       if (stage.taskDurationP50 > thresholds.maxP50 || stage.taskDurationP95 > thresholds.maxP95) return null;
       const coalesceTo = Math.max(1, Math.round(stage.taskCount / 10));
-      const fix = stage.shuffleReadBytes > 0
+      const reads = stageReads(stage);
+      const fix = reads === 'shuffle'
         ? `lower spark.sql.shuffle.partitions or .coalesce(${coalesceTo})`
         : `.coalesce(${coalesceTo})`;
       return {
-        type: 'tinyTask', stageId: stage.id, impactBand: 'info',
+        type: 'tinyTask', stageId: stage.id, impactBand: 'info', reads,
         metric: 'taskDurationP50', value: Math.round(stage.taskDurationP50),
         recommendation: `Many small tasks (${stage.taskCount}, P50 ${Math.round(stage.taskDurationP50)}ms): scheduler overhead may dominate. Try ${fix}.`,
-        remediation: stage.shuffleReadBytes > 0 ? [decreaseConf('spark.sql.shuffle.partitions')] : [],
+        remediation: reads === 'shuffle' ? [decreaseConf('spark.sql.shuffle.partitions')] : [],
       };
     },
     estimate(finding, ctx)                        {
@@ -1839,6 +2014,7 @@ export const DETECTORS = [
       const value = Math.round(gapSeconds);
       return {
         type: 'coldStart', stageId: null, impactBand: 'warning',
+        dynamicAllocation: dynamicAllocationOff(ctx.app) ? 'off' : 'on',
         metric: 'startupGapSeconds', value,
         recommendation: `The first stage waited ${value}s for an executor to become available: keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it doesn't scale up from zero.`,
         remediation: dynamicAllocationOff(ctx.app) ? [] : [increaseConf('spark.dynamicAllocation.minExecutors'), increaseConf('spark.dynamicAllocation.initialExecutors')],
@@ -1864,12 +2040,15 @@ export const DETECTORS = [
       if (!app || executorsAdded.length === 0 || app.startTime == null || app.endTime == null) return null;
       const appDuration = app.endTime - app.startTime;
       if (appDuration <= 0) return null;
-      // computePeakConcurrentCores (not executorsAdded.length/computeTotalCores): real concurrent
-      // capacity, not a cumulative sum that double-counts a churned-through executor against its
-      // replacement's (spot preemption, dynamicAllocation replacement).
+      // Capacity is what the run was allocated (cores x time alive, the figure behind
+      // metrics.allocation.coreHours), not peak concurrent cores x the whole run: under dynamic
+      // allocation or late-joining executors that is more than was ever held, and the idle figure
+      // would exceed the allocation. Null without executor cores.
+      const capacityCoreMs = allocatedCoreMs({ app, stages: ctx.stages, executors: { added: executorsAdded, removed: executorsRemoved } });
+      if (capacityCoreMs == null) return null;
+      // Peak concurrent cores (not computeTotalCores, a cumulative sum that double-counts a
+      // churned-through executor against its replacement): the cluster size the finding reports.
       const totalCores = computePeakConcurrentCores(app, executorsAdded, executorsRemoved);
-      if (totalCores <= 0) return null;
-      const capacityCoreMs = totalCores * appDuration;
       // Busy core-time (from the whole-run core-time-series, same signal memoryUtilization's
       // idleCores variant already uses), not executor lifetime: an executor that exists for the
       // whole run but sits fully idle must not score as 100% used. Missing runAggregates (older
@@ -1880,15 +2059,12 @@ export const DETECTORS = [
       if (utilization >= thresholds.minUtil) return null;
 
       // CPU-time-based utilization (sparkMeasure): metric only, no threshold.
-      let cpuUtilizationPct                = null;
-      if (totalCores > 0) {
-        // Null when no stage recorded CPU time (older Spark), the same rule as the CLI metrics block.
-        const cpuMs = totalExecutorCpuMs(ctx.stages.values());
-        cpuUtilizationPct = cpuMs == null ? null : Math.round((cpuMs / (appDuration * totalCores)) * 100);
-      }
+      // Null when no stage recorded CPU time (older Spark), the same rule as the CLI metrics block.
+      const cpuMs = totalExecutorCpuMs(ctx.stages.values());
+      const cpuUtilizationPct = cpuMs == null ? null : Math.round((cpuMs / capacityCoreMs) * 100);
 
       const value = Math.round(utilization * 100);
-      const fix = dynamicAllocationFix(app, 'consider reducing cluster size or enabling dynamic allocation',
+      const fix = idleCapacityFix(app, 'consider either reducing cluster size (spark.executor.instances) or enabling dynamic allocation',
         'dynamic allocation is already on, so consider reducing cluster size');
       return {
         type: 'utilization', stageId: null, impactBand: 'info',
@@ -1896,6 +2072,7 @@ export const DETECTORS = [
         utilizationFraction: utilization,
         appDurationMs: appDuration,
         totalCores,
+        allocatedCoreMs: capacityCoreMs,
         cpuUtilizationPct,
         recommendation: `Average executor utilization was only ${value}%: ${fix.text}.`,
         remediation: fix.remediation,
@@ -1903,13 +2080,17 @@ export const DETECTORS = [
     },
     estimate(finding)                        {
       const fraction = finding.utilizationFraction                      ;
-      const appDurationMs = finding.appDurationMs                      ;
-      const totalCores = finding.totalCores                      ;
-      if (fraction == null || appDurationMs == null || totalCores == null) {
+      const allocatedMs = finding.allocatedCoreMs                      ;
+      if (fraction == null || allocatedMs == null) {
         return costOnly('measured');
       }
-      const idleCoreHours = (1 - fraction) * appDurationMs * totalCores / 3.6e6;
-      return costOnly('measured', { value: idleCoreHours, unit: 'coreHours', idle: true });
+      // The same allocated-minus-busy figure as `rawWaste`, in core-milliseconds, next to (never in)
+      // coreTimeMs: that field is busy task time a fix removes, this is capacity no task used.
+      const idleMs = (1 - fraction) * allocatedMs;
+      return {
+        ...costOnly('measured', { value: idleMs / MS_PER_CORE_HOUR, unit: 'coreHours', idle: true }),
+        idleCoreTimeMs: { low: idleMs, high: idleMs },
+      };
     },
   }),
   defineAppDetector({
@@ -1934,25 +2115,26 @@ export const DETECTORS = [
       // cumulative sum or count double-counts a churned-through executor against its replacement's
       // (spot preemption, dynamicAllocation replacement), inflating idle-rate and waste-model figures.
       const peakExecutors = computePeakConcurrentExecutorCount(executorsAdded, executorsRemoved);
-      const totalCores = computePeakConcurrentCores(app, executorsAdded, executorsRemoved);
-      // Hoisted above 1a (also 1b/1c's input) so the idle-cores finding carries the allocated
-      // memory its MB-seconds estimate needs.
       const allocatedMB = app.resources?.executor?.memoryMB ?? null;
 
       // ── 1a idle-cores rate ────────────────────────────────────────────────
       const busyCoreMs = runAggregates?.busyCoreMs;
-      if (busyCoreMs != null && totalCores > 0) {
-        const capacityCoreMs = totalCores * appDurationMs;
-        const idleRate = capacityCoreMs > 0 ? 1 - (busyCoreMs / capacityCoreMs) : 0;
+      // Allocated core-time, as for `utilization`, so the two findings of one idle condition agree.
+      const allocationInput = { app, stages, executors: { added: executorsAdded, removed: executorsRemoved } };
+      const capacityCoreMs = allocatedCoreMs(allocationInput);
+      if (busyCoreMs != null && capacityCoreMs != null) {
+        const idleRate = 1 - (busyCoreMs / capacityCoreMs);
         if (idleRate > thresholds.idleCoreWarn) {
           const value = Math.round(idleRate * 100);
-          const fix = dynamicAllocationFix(app, 'reduce cluster size or enable dynamic allocation',
+          const { memoryGbHours } = computeAllocation(allocationInput);
+          const fix = idleCapacityFix(app, 'either reduce cluster size (spark.executor.instances) or enable dynamic allocation',
             'dynamic allocation is already on, so reduce cluster size');
           out.push({
             type: 'memoryUtilization', variant: 'idleCores', stageId: null,
             impactBand: 'warning', metric: 'idleCoreRate', value,
-            // Raw (unrounded) rate plus sizing inputs for the impact estimator: `value` is rounded pct.
-            idleRateFraction: idleRate, allocatedMB, peakExecutors, appDurationMs,
+            // Raw (unrounded) rate plus the allocated memory-time for the impact estimator: `value` is rounded pct.
+            idleRateFraction: idleRate,
+            allocatedMBSeconds: memoryGbHours != null ? memoryGbHours * 1024 * 3600 : null,
             recommendation: `${value}% of available core-time ran no task: ${fix.text}.`,
             remediation: fix.remediation,
           });
@@ -2038,13 +2220,11 @@ export const DETECTORS = [
         return costOnly('measured', { value: finding.value, unit: 'mbSeconds' });
       }
       if (finding.variant === 'idleCores') {
-        // Idle core-time priced as memory held but unused: the same MB-seconds unit as wasteModel, so comparable.
+        // Idle core-time priced as memory held but unused: the allocated memory-seconds times the idle rate.
         const idleRateFraction = finding.idleRateFraction                      ;
-        const allocatedMB = finding.allocatedMB                      ;
-        const peakExecutors = finding.peakExecutors                      ;
-        const appDurationMs = finding.appDurationMs                      ;
-        if (idleRateFraction != null && allocatedMB != null && peakExecutors != null && appDurationMs != null) {
-          const wastedMBSeconds = idleRateFraction * allocatedMB * peakExecutors * (appDurationMs / 1000);
+        const allocatedMBSeconds = finding.allocatedMBSeconds                             ;
+        if (idleRateFraction != null && allocatedMBSeconds != null) {
+          const wastedMBSeconds = idleRateFraction * allocatedMBSeconds;
           return costOnly('modeled', { value: wastedMBSeconds, unit: 'mbSeconds' });
         }
         return costOnly('modeled');
@@ -2196,14 +2376,19 @@ export const DETECTORS = [
       if (!impactBand) return null;
 
       const pct = Math.round(shortLivedPct * 100);
+      const daOff = dynamicAllocationOff(app);
       return {
         type: 'autoscalingChurn', stageId: null, impactBand,
         metric: 'shortLivedExecutorPct', value: pct,
         // Raw count behind the percentage, for the impact estimator's startup-overhead figure.
         shortLivedExecutorCount: shortLivedCount,
         confidence: autoscalingChurnConfidence(shortLivedPct, thresholds.warningPct, thresholds.criticalPct),
-        recommendation: `${pct}% of executors ran for under 2 minutes before being removed: this looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
-        remediation: [
+        // No dynamic-allocation property has an effect when the run's conf turns it off (as for coldStart).
+        dynamicAllocation: daOff ? 'off' : 'on',
+        recommendation: daOff
+          ? `${pct}% of executors ran for under 2 minutes before being removed: dynamic allocation is off, so the churn comes from executors lost or preempted, not from autoscaling bounds; check the cluster manager's preemption and executor-loss logs.`
+          : `${pct}% of executors ran for under 2 minutes before being removed: this looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
+        remediation: daOff ? [] : [
           increaseConf('spark.dynamicAllocation.executorIdleTimeout'),
           decreaseConf('spark.dynamicAllocation.minExecutors'),
           increaseConf('spark.dynamicAllocation.maxExecutors'),
@@ -2705,14 +2890,21 @@ export const DETECTORS = [
               || (smaller < broadcastTiers[3] && larger > comparisonTiers[2]);
             if (fires) {
               const contributors = [...boundarySizeContributors(childA), ...boundarySizeContributors(childB)];
+              // A threshold that already admits the smaller side cannot be what stopped the broadcast.
+              const threshold = effectiveBroadcastThreshold(ctx.app);
+              const broadcastThreshold                     = threshold != null && threshold < 0 ? 'disabled'
+                : threshold != null && threshold > smaller ? 'notLimiting' : 'limits';
+              const raiseAdvice = 'Consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.';
               out.push({
                 type: 'underBroadcast', executionId: sqlExec.id, stageIds: unionStageIds(contributors, fallbackStageIds),
                 // resolvePlanTree always sets id; safe downstream of it.
                 planNodeIds: contributors.map((n) => n.id ).filter(Boolean),
                 impactBand: 'info', metric: 'smallerSideBytes', value: smaller,
-                largerSideBytes: larger,
-                recommendation: `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is well under the broadcast threshold relative to the larger side (${formatBytes(larger)}): this could have been a broadcast join. Consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.`,
-                remediation: [increaseConf('spark.sql.autoBroadcastJoinThreshold')],
+                largerSideBytes: larger, broadcastThreshold,
+                recommendation: broadcastThreshold === 'notLimiting'
+                  ? `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is under the effective spark.sql.autoBroadcastJoinThreshold (${formatBytes(threshold )}) yet was not broadcast (the larger side is ${formatBytes(larger)}), so the threshold is not what stopped it: missing table statistics or a join type that cannot broadcast usually is. Consider a broadcast() hint or collecting statistics (ANALYZE TABLE).`
+                  : `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is well under the broadcast threshold relative to the larger side (${formatBytes(larger)}): this could have been a broadcast join. ${raiseAdvice}`,
+                remediation: broadcastThreshold === 'notLimiting' ? [] : [increaseConf('spark.sql.autoBroadcastJoinThreshold')],
               });
             }
           }
@@ -2724,17 +2916,23 @@ export const DETECTORS = [
             // (node.stageIds always empty in real data); its child carries the executor-side
             // metrics, so only the child unions in.
             const child = (node.children ?? [])[0];
-            const autoBroadcastOff = ctx.app?.config?.['spark.sql.autoBroadcastJoinThreshold']?.trim() === '-1';
+            // A threshold below the broadcast cannot have admitted it, so a hint forced it.
+            const threshold = effectiveBroadcastThreshold(ctx.app);
+            const autoBroadcastOff = threshold != null && threshold < 0;
+            const hintForced = autoBroadcastOff || (threshold != null && m.value > threshold);
+            const broadcastThreshold                     = autoBroadcastOff ? 'disabled' : hintForced ? 'notLimiting' : 'limits';
             out.push({
               type: 'overBroadcast', executionId: sqlExec.id,
               stageIds: unionStageIds(child ? [child] : [], fallbackStageIds),
               // resolvePlanTree always sets id; safe downstream of it.
               planNodeIds: [node.id ].filter(Boolean),
-              impactBand: 'warning', metric: 'broadcastBytes', value: m.value,
+              impactBand: 'warning', metric: 'broadcastBytes', value: m.value, broadcastThreshold,
               recommendation: autoBroadcastOff
                 ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: automatic broadcast is already disabled, so remove the broadcast() hint that forced it.`
-                : `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
-              remediation: autoBroadcastOff ? [] : [decreaseConf('spark.sql.autoBroadcastJoinThreshold')],
+                : hintForced
+                  ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: spark.sql.autoBroadcastJoinThreshold (${formatBytes(threshold )}) is below it, so a broadcast() hint forced it: remove the hint.`
+                  : `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
+              remediation: hintForced ? [] : [decreaseConf('spark.sql.autoBroadcastJoinThreshold')],
             });
           }
         }