sparkforensics-server 0.5.0 → 0.5.2

package/vendor-core/detectors.js

@@ -1,4 +1,5 @@
 import { pathBasename, formatBytes, IMPACT_BAND_ORDER } from './format-utils.js';
+import { medianOfSorted } from './median.js';
 import { shareLabel } from './finding-presentation.js';
 import { scanRelationId } from './plan-summary.js';
 import { computePeakConcurrentCores, computePeakConcurrentExecutorCount } from './core-count.js';
@@ -21,7 +22,7 @@ import { cyrb53 } from './string-hash.js';
 import { decreaseConf, increaseConf, setConf } from './remediation.js';
 import { MAX_FAILURE_GROUPS, describeTaskFailure,                       } from './task-failure.js';
                                                                                                
-                                                                                                                      
+                                                                                                                                                                                     
 
 const MB = 1024 * 1024;
 const GB = 1024 * MB;
@@ -543,7 +544,7 @@ function maxMedianRatio(
 )                                                       {
   if (samples.length < 3) return null;
   const vals = samples.map(s => s.value).sort((a, b) => a - b);
-  const median = vals[Math.floor(vals.length / 2)];
+  const median = medianOfSorted(vals);
   if (median <= 0) return null;
   const top = samples.reduce((a, b) => (b.value > a.value ? b : a));
   return { key: top.key, ratio: top.value / median, value: top.value };
@@ -655,14 +656,36 @@ function tailClaimFloorMs(claim           , stageId        , ctx             )
   return tailClaimImpact(claim, stageId, ctx.impact).wallClock?.high ?? claim.wasteMs;
 }
 
-// A setting is only a fix when the run's logged conf doesn't already have it: a run that set it
-// needs another remedy. Only explicitly logged properties count; Spark's unlogged version
-// defaults are not modeled. Booleans compare case-insensitively, as Spark parses them.
+// Spark's default for the properties a detector's fix reads or suggests, for a run that did not log
+// them. Undefined when the property is not modeled or its default depends on a version the log did
+// not record. A property is modeled only where its default has made a suggestion a no-op; every
+// other property a detector suggests defaults off, or to a value other than the suggested one.
+function versionDefault(app                    , key        )                     {
+  if (key === 'spark.sql.shuffle.partitions') return '200';
+  if (key === 'spark.sql.autoBroadcastJoinThreshold') return String(10 * MB);
+  const version = /^(\d+)\.(\d+)/.exec(app?.sparkVersion ?? '');
+  if (version == null) return undefined;
+  const [major, minor] = [Number(version[1]), Number(version[2])];
+  if (key === 'spark.sql.adaptive.enabled') return major > 3 || (major === 3 && minor >= 2) ? 'true' : 'false';
+  if (key === 'spark.sql.adaptive.skewJoin.enabled' || key === 'spark.sql.adaptive.coalescePartitions.enabled') {
+    return major >= 3 ? 'true' : undefined;
+  }
+  return undefined;
+}
+
+// The run's effective value of a property: the logged one, else versionDefault's.
+function effectiveConf(app                    , key        )                     {
+  return (app?.config?.[key] ?? versionDefault(app, key))?.trim();
+}
+
+// A setting is only a fix when the run's effective conf doesn't already have it: a run that set it,
+// or whose Spark version defaults to it, needs another remedy. Booleans compare case-insensitively,
+// as Spark parses them.
 function loggedAs(app                    , key        , suggested                  )          {
-  const logged = app?.config?.[key]?.trim();
+  const effective = effectiveConf(app, key);
   return typeof suggested === 'boolean'
-    ? logged?.toLowerCase() === String(suggested)
-    : logged === suggested;
+    ? effective?.toLowerCase() === String(suggested)
+    : effective === suggested;
 }
 
 function setConfUnlessLogged(app                    , key        , suggested                  )                {
@@ -675,18 +698,67 @@ function switchFix(on         , key        , suggested                  , recomm
   return on ? { text: alreadyOn, remediation: [] } : { text: recommend, remediation: [setConf(key, suggested)] };
 }
 
+const SKEW_KEY_REMEDY = 'salt the key or repartition on a better key';
+
+// What a stage reads, by the dominant side: a stage that reads a shuffle and mostly input files is
+// an input stage for any fix that sizes a shuffle's reduce side.
+function stageReads(stage                                                        )             {
+  if (stage.shuffleReadBytes > 0 && stage.shuffleReadBytes >= stage.inputBytes) return 'shuffle';
+  return stage.inputBytes > 0 ? 'input' : 'other';
+}
+
+// Whether the run's Spark predates 3.0, which has no AQE skew-join handling to suggest.
+function predatesAqeSkewJoin(app                    )          {
+  const major = /^(\d+)\./.exec(app?.sparkVersion ?? '');
+  return major != null && Number(major[1]) < 3;
+}
+
+// What a skewed stage reads, which decides whether AQE skew-join handling can act on it. It splits
+// skewed partitions on the shuffle-read side of a sort-merge or shuffled-hash join, so only a stage
+// that reads a shuffle, in a SQL execution whose plan has such a join, is 'shuffleJoin'. A plan does
+// not say which stage runs which join (a node's stageIds are empty in real logs), so the join is
+// matched per execution. 'inputScan' reads mostly files (its tasks differ in input size);
+// 'other' is any stage the plan cannot tie to a join (an aggregation's shuffle, no plan, no input),
+// and any stage of a run on Spark before 3.0 (skewFix).
+function skewOrigin(stage               , ctx             , shuffleEvidence         )                                        {
+  if (shuffleEvidence || (stage.shuffleReadBytes > 0 && stage.shuffleReadBytes >= stage.inputBytes)) {
+    let hasJoin = false;
+    const plan = stage.sqlExecutionId != null ? ctx.sql.get(stage.sqlExecutionId)?.planTree : null;
+    walkPlanTree(plan, (node) => { if (SKEW_JOIN_NODES.has(node.name)) hasJoin = true; });
+    return hasJoin ? 'shuffleJoin' : 'other';
+  }
+  return stage.inputBytes > 0 ? 'inputScan' : 'other';
+}
+const SKEW_JOIN_NODES = new Set(['SortMergeJoin', 'ShuffledHashJoin']);
+
+// The skew finding's fix for the stage's origin. Join-driven skew gets AQE skew-join handling,
+// unless the run's effective conf already has it; uneven input gets the file-size remedy.
+function skewFix(stage               , ctx             , shuffleEvidence = false)                                                                   {
+  const origin = skewOrigin(stage, ctx, shuffleEvidence);
+  if (origin === 'inputScan') {
+    return {
+      origin,
+      text: 'the stage reads uneven input files: compact small files or split large ones (lower spark.sql.files.maxPartitionBytes)',
+      remediation: [decreaseConf('spark.sql.files.maxPartitionBytes')],
+    };
+  }
+  if (origin === 'other') return { origin, text: SKEW_KEY_REMEDY, remediation: [] };
+  // Before Spark 3.0 there is no AQE skew-join handling, so the stage gets the generic advice.
+  if (predatesAqeSkewJoin(ctx.app)) return { origin: 'other', text: SKEW_KEY_REMEDY, remediation: [] };
+  return { origin, ...skewJoinFix(ctx.app) };
+}
+
 function skewJoinFix(app                    )                                               {
   const key = 'spark.sql.adaptive.skewJoin.enabled';
-  const remedy = 'salt the key or repartition on a better key';
   if (loggedAs(app, 'spark.sql.adaptive.enabled', false)) {
     return {
-      text: `AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise ${remedy}`,
+      text: `AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise ${SKEW_KEY_REMEDY}`,
       remediation: [setConf('spark.sql.adaptive.enabled', true), ...setConfUnlessLogged(app, key, true)],
     };
   }
   return switchFix(loggedAs(app, key, true), key, true,
-    `for join-driven skew, enable AQE skew-join handling (${key}); otherwise ${remedy}`,
-    `AQE skew-join handling is already on, so ${remedy}`);
+    `for join-driven skew, enable AQE skew-join handling (${key}); otherwise ${SKEW_KEY_REMEDY}`,
+    `AQE skew-join handling is already on, so ${SKEW_KEY_REMEDY}`);
 }
 
 // The resources flag is read from the same property as the logged conf.
@@ -695,31 +767,75 @@ function dynamicAllocationFix(app             , recommend        , alreadyOn
   return switchFix(app.resources?.dynamicAllocationEnabled === true || loggedAs(app, key, true), key, true, recommend, alreadyOn);
 }
 
-// The run's logged spark.sql.shuffle.partitions as a count, or null when unlogged or not a count.
-function loggedShufflePartitions(app                    )                {
-  const logged = app?.config?.['spark.sql.shuffle.partitions']?.trim();
-  return logged != null && /^\d+$/.test(logged) ? Number(logged) : null;
+// The run's effective spark.sql.shuffle.partitions (logged, else Spark's 200) as a count, or null
+// when the logged value is not a count.
+function effectiveShufflePartitions(app                    )                {
+  const value = effectiveConf(app, 'spark.sql.shuffle.partitions');
+  return value != null && /^\d+$/.test(value) ? Number(value) : null;
+}
+
+// Whether partition-count advice fits a shuffle-reading stage, from the effective conf and the
+// stage's own task sizes. The partition count that brings each shuffle partition down to the ideal
+// size is per stage; the property is job-wide. 'sufficient': the stage's tasks are already at or
+// under the ideal size. 'aqeCoalesced': AQE merged the property's partitions into fewer tasks, so
+// the advisory partition size is the lever. 'ownPartitioning': the property already gives at least
+// the count needed, so the stage's own repartition(n) or RDD parallelism limits it. Otherwise the
+// property limits the stage: 'raise'.
+function shufflePartitionCase(stage               , app                    )                                                                          {
+  const count = effectiveShufflePartitions(app);
+  const needed = Math.ceil(stage.shuffleReadBytes / IDEAL_BYTES_PER_PARTITION_TASK);
+  const tasks = Math.max(1, stage.taskCount);
+  let partitions                    = 'raise';
+  if (stage.shuffleReadBytes / tasks <= IDEAL_BYTES_PER_PARTITION_TASK) partitions = 'sufficient';
+  else if (count != null && stage.taskCount < count
+    && loggedAs(app, 'spark.sql.adaptive.enabled', true) && loggedAs(app, 'spark.sql.adaptive.coalescePartitions.enabled', true)) partitions = 'aqeCoalesced';
+  else if (count != null && count >= needed) partitions = 'ownPartitioning';
+  return { partitions, count, needed };
 }
 
-// lowShuffleParallelism's fix. The partition count that brings each shuffle partition down to the
-// ideal size, as the estimate models it, is per stage; the property is job-wide. A logged value at
-// or above it means the property is not what limits that stage (a repartition(n) or an RDD
-// shuffle is), so the text points at the stage's own partitioning and no property is suggested.
-// Unlogged, the count is not a safe value: null.
-function lowShuffleParallelismFix(app                    , needed        )                                               {
-  const logged = loggedShufflePartitions(app);
-  if (logged != null && logged >= needed) {
+const ADVISORY_PARTITION_SIZE_KEY = 'spark.sql.adaptive.advisoryPartitionSizeInBytes';
+
+// lowShuffleParallelism's fix. Unlike the shuffle finding it fires on stages whose tasks are larger
+// than the ideal size, so its cases are 'raise', 'aqeCoalesced' and 'ownPartitioning'.
+function lowShuffleParallelismFix(stage               , app                    )                                                                              {
+  const { partitions, count, needed } = shufflePartitionCase(stage, app);
+  if (partitions === 'aqeCoalesced') {
     return {
-      text: `spark.sql.shuffle.partitions is already ${logged}, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller`,
+      partitions,
+      text: `AQE coalesced the shuffle into ${stage.taskCount} tasks (from ${count} configured partitions): lower ${ADVISORY_PARTITION_SIZE_KEY} so each partition is smaller`,
+      remediation: [decreaseConf(ADVISORY_PARTITION_SIZE_KEY)],
+    };
+  }
+  if (partitions === 'ownPartitioning') {
+    return {
+      partitions,
+      text: `spark.sql.shuffle.partitions is already ${count}, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller`,
       remediation: [],
     };
   }
   return {
+    partitions,
     text: 'raise spark.sql.shuffle.partitions so each partition is smaller',
-    remediation: [increaseConf('spark.sql.shuffle.partitions', logged == null ? null : needed)],
+    remediation: [increaseConf('spark.sql.shuffle.partitions', count == null ? null : needed)],
   };
 }
 
+// A Spark byte-size property as bytes (a plain number, or with a k/m/g/t suffix, optionally 'b'), or
+// null when it is not one. Negative numbers pass through: -1 turns auto-broadcast off.
+function parseSparkBytes(value                    )                {
+  const m = /^(-?\d+(?:\.\d+)?)\s*([kmgt]?)b?$/i.exec(value?.trim() ?? '');
+  if (m == null) return null;
+  return m[2] === '' ? Number(m[1]) : Number(m[1]) * 1024 ** ('kmgt'.indexOf(m[2].toLowerCase()) + 1);
+}
+
+// The run's effective spark.sql.autoBroadcastJoinThreshold in bytes (logged, else Spark's 10 MiB);
+// negative when auto-broadcast is disabled, null when the logged value is not a size.
+function effectiveBroadcastThreshold(app                    )                {
+  const raw = effectiveConf(app, 'spark.sql.autoBroadcastJoinThreshold');
+  if (raw != null && /^-\d+$/.test(raw)) return -1;
+  return parseSparkBytes(raw);
+}
+
 // No dynamic-allocation property has an effect on a run whose logged conf turns it off.
 function dynamicAllocationOff(app                    )          {
   return app?.resources?.dynamicAllocationEnabled === false || app?.config?.['spark.dynamicAllocation.enabled']?.trim().toLowerCase() === 'false';
@@ -1061,9 +1177,9 @@ export const DETECTORS = [
       const floorWasteMs = tailClaimFloorMs(skewTailClaim(stage, metric === 'P95/median'), stage.id, ctx);
       if (!meetsRuntimeFloor(floorWasteMs, appDurationMs(ctx.app), thresholds.floorPctWarn)) return null;
       const value = Math.round(ratio * 10) / 10;
-      const fix = skewJoinFix(ctx.app);
+      const fix = skewFix(stage, ctx);
       return {
-        type: 'skew', stageId: stage.id,
+        type: 'skew', stageId: stage.id, origin: fix.origin,
         impactBand: 'warning',
         metric, value,
         confidence: skewConfidence(ratio, thresholds.ratioWarn),
@@ -1188,12 +1304,21 @@ export const DETECTORS = [
       const bytes = stage.shuffleReadBytes;
       if (bytes <= thresholds.minBytes) return null;
       if (stageBelowRuntimeFloor(stage, ctx, thresholds.stageFloorPct)) return null;
+      const { partitions, count } = shufflePartitionCase(stage, ctx.app);
+      const perTask = formatBytes(bytes / Math.max(1, stage.taskCount));
+      const fix = partitions === 'sufficient'
+        ? { text: `its ${stage.taskCount} tasks already read about ${perTask} each, so more partitions will not help: consider a broadcast join for the smaller side`, remediation: [] }
+        : partitions === 'aqeCoalesced'
+          ? { text: `AQE coalesced it into ${stage.taskCount} tasks (from ${count} configured partitions): consider lowering ${ADVISORY_PARTITION_SIZE_KEY} or adding a broadcast join`, remediation: [decreaseConf(ADVISORY_PARTITION_SIZE_KEY)] }
+          : partitions === 'ownPartitioning'
+            ? { text: `spark.sql.shuffle.partitions is already ${count}, so consider raising this stage's own partition count (its repartition(n) or RDD parallelism) or adding a broadcast join`, remediation: [] }
+            : { text: 'consider increasing spark.sql.shuffle.partitions or adding a broadcast join', remediation: [increaseConf('spark.sql.shuffle.partitions')] };
       return {
         type: 'shuffle', stageId: stage.id,
-        impactBand: 'info',
+        impactBand: 'info', partitions,
         metric: 'shuffleReadBytes', value: bytes,
-        recommendation: `${formatBytes(bytes)} shuffled in this stage: consider increasing spark.sql.shuffle.partitions or adding a broadcast join.`,
-        remediation: [increaseConf('spark.sql.shuffle.partitions')],
+        recommendation: `${formatBytes(bytes)} shuffled in this stage: ${fix.text}.`,
+        remediation: fix.remediation,
       };
     },
     estimate(finding, ctx)                        {
@@ -1225,19 +1350,19 @@ export const DETECTORS = [
         const ratioText = p50 > 0
           ? `${Math.round(max / p50 * 10) / 10}× the median (${formatBytes(p50)})`
           : 'far larger than the median, which is effectively empty';
-        const fix = skewJoinFix(ctx.app);
+        const fix = skewFix(stage, ctx, true);
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
-          rule: 'shufflePartitionSkew', metric: 'shuffleReadMax', value: max,
+          rule: 'shufflePartitionSkew', origin: fix.origin, metric: 'shuffleReadMax', value: max,
           recommendation: `The largest shuffle partition (${formatBytes(max)}) is ${ratioText}: ${fix.text}.`,
           remediation: fix.remediation,
         });
       }
       if (total >= thresholds.lowParTotalBytes && taskCount <= thresholds.lowParMaxTasks) {
-        const fix = lowShuffleParallelismFix(ctx.app, Math.ceil(total / IDEAL_BYTES_PER_PARTITION_TASK));
+        const fix = lowShuffleParallelismFix(stage, ctx.app);
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
-          rule: 'lowShuffleParallelism', metric: 'taskCount', value: taskCount,
+          rule: 'lowShuffleParallelism', partitions: fix.partitions, metric: 'taskCount', value: taskCount,
           recommendation: `${Math.round(total / GB * 10) / 10} GB of shuffle spread over only ${taskCount} tasks: ${fix.text}.`,
           remediation: fix.remediation,
         });
@@ -1296,8 +1421,11 @@ export const DETECTORS = [
       const classified = cls === 'skew' || cls === 'volume';
       const mag = computeSpillMagnitude(stage, thresholds);
       const impactBand = 'warning';
+      // shuffle.partitions sizes a shuffle's reduce side: a stage that reads no shuffle gets memory only.
+      const reads = stageReads(stage);
+      const partitionAdvice = reads === 'shuffle';
       return {
-        type: 'spill', stageId: stage.id, impactBand,
+        type: 'spill', stageId: stage.id, impactBand, reads,
         spillMagnitude: mag?.magnitude,
         metric: 'memoryBytesSpilled', value: stage.memoryBytesSpilled,
         confidence: classified ? 'medium' : 'low',
@@ -1306,8 +1434,10 @@ export const DETECTORS = [
           : 'Spill cause could not be classified: inspect per-task spill metrics in the Spark UI before acting.',
         recommendation: cls === 'skew'
           ? `${formatBytes(stage.memoryBytesSpilled)} spilled, skew-driven: fix task skew first; adding memory will not help.`
-          : `${formatBytes(stage.memoryBytesSpilled)} spilled: raise spark.sql.shuffle.partitions or increase executor memory.`,
-        remediation: cls === 'skew' ? [] : [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.executor.memory')],
+          : partitionAdvice
+            ? `${formatBytes(stage.memoryBytesSpilled)} spilled: raise spark.sql.shuffle.partitions or increase executor memory.`
+            : `${formatBytes(stage.memoryBytesSpilled)} spilled: this stage reads no shuffle, so increase executor memory or process less data per task.`,
+        remediation: cls === 'skew' ? [] : [...(partitionAdvice ? [increaseConf('spark.sql.shuffle.partitions')] : []), increaseConf('spark.executor.memory')],
       };
     },
     estimate(finding, ctx)                        {
@@ -1419,7 +1549,7 @@ export const DETECTORS = [
       if (hosts.length >= thresholds.minHosts) {
         const means = hosts.map(h => ({ host: h.host, taskCount: h.taskCount, mean: h.totalDuration / h.taskCount }));
         const sorted = [...means].map(h => h.mean).sort((a, b) => a - b);
-        const overallMedian = sorted[Math.floor(sorted.length / 2)];
+        const overallMedian = medianOfSorted(sorted);
         if (overallMedian > 0) {
           for (const h of means) {
             const ratio = h.mean / overallMedian;
@@ -1532,11 +1662,21 @@ export const DETECTORS = [
       const impactBand = durationMinutes >= t.infoMin ? 'info' : null;
       if (!impactBand) return null;
       const value = Math.round(durationMinutes * 10) / 10;
+      // shuffle.partitions and default.parallelism size a shuffle's reduce side: a stage that reads
+      // no shuffle gets the input-partitioning remedy instead.
+      const reads = stageReads(stage);
+      const recommendation = reads === 'shuffle'
+        ? `This stage ran ${value} minutes with no more specific cause flagged: often a partition-count problem, raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.`
+        : reads === 'input'
+          ? `This stage ran ${value} minutes with no more specific cause flagged and reads mostly input files: often too few or too uneven input partitions, so check input file sizes and lower spark.sql.files.maxPartitionBytes, or look for a large per-task data volume driving heavy spill.`
+          : `This stage ran ${value} minutes with no more specific cause flagged and reads neither shuffle nor input files: check what it computes and for a large per-task data volume driving heavy spill.`;
       return {
         type: 'stageSlowness', stageId: stage.id, impactBand,
-        metric: 'stageDurationMinutes', value,
-        recommendation: `This stage ran ${value} minutes with no more specific cause flagged: often a partition-count problem, raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.`,
-        remediation: [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.default.parallelism')],
+        metric: 'stageDurationMinutes', value, reads,
+        recommendation,
+        remediation: reads === 'shuffle'
+          ? [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.default.parallelism')]
+          : reads === 'input' ? [decreaseConf('spark.sql.files.maxPartitionBytes')] : [],
       };
     },
     estimate(finding, ctx)                        {
@@ -1664,8 +1804,10 @@ export const DETECTORS = [
       const detail = useSpeculativeMetric
         ? `${value} speculative attempt${value === 1 ? '' : 's'} discarded`
         : `${value}% of tasks straggled`;
+      // The skew advice fits only a stage that reads a shuffle feeding a join (skewFix).
+      const fix = skewFix(stage, ctx);
       return {
-        type: 'straggler', stageId: stage.id, impactBand,
+        type: 'straggler', stageId: stage.id, impactBand, origin: fix.origin,
         metric: useSpeculativeMetric ? 'speculativeTasks' : 'stragglerShare',
         value,
         unit: useSpeculativeMetric ? 'count' : 'pct',
@@ -1675,7 +1817,8 @@ export const DETECTORS = [
           ? stragglerConfidence(speculativeShare, thresholds.warnPct, thresholds.critPct)
           : stragglerConfidence(stragglerShare, thresholds.shareWarn, thresholds.critPct),
         validationRequired: `Warning needs at least ${shareLabel(thresholds.floorPctWarn)} of run time at stake, critical ${shareLabel(thresholds.floorPctCrit)}.`,
-        recommendation: `${detail}: rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that's a candidate for AQE's skew-join handling.`,
+        recommendation: `${detail}: rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if uneven data is the cause, ${fix.text}.`,
+        remediation: fix.remediation,
       };
     },
     estimate(finding, ctx)                        {
@@ -1755,14 +1898,15 @@ export const DETECTORS = [
       if (stageBelowRuntimeFloor(stage, ctx, thresholds.stageFloorPct)) return null;
       if (stage.taskDurationP50 > thresholds.maxP50 || stage.taskDurationP95 > thresholds.maxP95) return null;
       const coalesceTo = Math.max(1, Math.round(stage.taskCount / 10));
-      const fix = stage.shuffleReadBytes > 0
+      const reads = stageReads(stage);
+      const fix = reads === 'shuffle'
         ? `lower spark.sql.shuffle.partitions or .coalesce(${coalesceTo})`
         : `.coalesce(${coalesceTo})`;
       return {
-        type: 'tinyTask', stageId: stage.id, impactBand: 'info',
+        type: 'tinyTask', stageId: stage.id, impactBand: 'info', reads,
         metric: 'taskDurationP50', value: Math.round(stage.taskDurationP50),
         recommendation: `Many small tasks (${stage.taskCount}, P50 ${Math.round(stage.taskDurationP50)}ms): scheduler overhead may dominate. Try ${fix}.`,
-        remediation: stage.shuffleReadBytes > 0 ? [decreaseConf('spark.sql.shuffle.partitions')] : [],
+        remediation: reads === 'shuffle' ? [decreaseConf('spark.sql.shuffle.partitions')] : [],
       };
     },
     estimate(finding, ctx)                        {
@@ -1839,6 +1983,7 @@ export const DETECTORS = [
       const value = Math.round(gapSeconds);
       return {
         type: 'coldStart', stageId: null, impactBand: 'warning',
+        dynamicAllocation: dynamicAllocationOff(ctx.app) ? 'off' : 'on',
         metric: 'startupGapSeconds', value,
         recommendation: `The first stage waited ${value}s for an executor to become available: keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it doesn't scale up from zero.`,
         remediation: dynamicAllocationOff(ctx.app) ? [] : [increaseConf('spark.dynamicAllocation.minExecutors'), increaseConf('spark.dynamicAllocation.initialExecutors')],
@@ -2196,14 +2341,19 @@ export const DETECTORS = [
       if (!impactBand) return null;
 
       const pct = Math.round(shortLivedPct * 100);
+      const daOff = dynamicAllocationOff(app);
       return {
         type: 'autoscalingChurn', stageId: null, impactBand,
         metric: 'shortLivedExecutorPct', value: pct,
         // Raw count behind the percentage, for the impact estimator's startup-overhead figure.
         shortLivedExecutorCount: shortLivedCount,
         confidence: autoscalingChurnConfidence(shortLivedPct, thresholds.warningPct, thresholds.criticalPct),
-        recommendation: `${pct}% of executors ran for under 2 minutes before being removed: this looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
-        remediation: [
+        // No dynamic-allocation property has an effect when the run's conf turns it off (as for coldStart).
+        dynamicAllocation: daOff ? 'off' : 'on',
+        recommendation: daOff
+          ? `${pct}% of executors ran for under 2 minutes before being removed: dynamic allocation is off, so the churn comes from executors lost or preempted, not from autoscaling bounds; check the cluster manager's preemption and executor-loss logs.`
+          : `${pct}% of executors ran for under 2 minutes before being removed: this looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
+        remediation: daOff ? [] : [
           increaseConf('spark.dynamicAllocation.executorIdleTimeout'),
           decreaseConf('spark.dynamicAllocation.minExecutors'),
           increaseConf('spark.dynamicAllocation.maxExecutors'),
@@ -2705,14 +2855,21 @@ export const DETECTORS = [
               || (smaller < broadcastTiers[3] && larger > comparisonTiers[2]);
             if (fires) {
               const contributors = [...boundarySizeContributors(childA), ...boundarySizeContributors(childB)];
+              // A threshold that already admits the smaller side cannot be what stopped the broadcast.
+              const threshold = effectiveBroadcastThreshold(ctx.app);
+              const broadcastThreshold                     = threshold != null && threshold < 0 ? 'disabled'
+                : threshold != null && threshold > smaller ? 'notLimiting' : 'limits';
+              const raiseAdvice = 'Consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.';
               out.push({
                 type: 'underBroadcast', executionId: sqlExec.id, stageIds: unionStageIds(contributors, fallbackStageIds),
                 // resolvePlanTree always sets id; safe downstream of it.
                 planNodeIds: contributors.map((n) => n.id ).filter(Boolean),
                 impactBand: 'info', metric: 'smallerSideBytes', value: smaller,
-                largerSideBytes: larger,
-                recommendation: `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is well under the broadcast threshold relative to the larger side (${formatBytes(larger)}): this could have been a broadcast join. Consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.`,
-                remediation: [increaseConf('spark.sql.autoBroadcastJoinThreshold')],
+                largerSideBytes: larger, broadcastThreshold,
+                recommendation: broadcastThreshold === 'notLimiting'
+                  ? `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is under the effective spark.sql.autoBroadcastJoinThreshold (${formatBytes(threshold )}) yet was not broadcast (the larger side is ${formatBytes(larger)}), so the threshold is not what stopped it: missing table statistics or a join type that cannot broadcast usually is. Consider a broadcast() hint or collecting statistics (ANALYZE TABLE).`
+                  : `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is well under the broadcast threshold relative to the larger side (${formatBytes(larger)}): this could have been a broadcast join. ${raiseAdvice}`,
+                remediation: broadcastThreshold === 'notLimiting' ? [] : [increaseConf('spark.sql.autoBroadcastJoinThreshold')],
               });
             }
           }
@@ -2724,17 +2881,23 @@ export const DETECTORS = [
             // (node.stageIds always empty in real data); its child carries the executor-side
             // metrics, so only the child unions in.
             const child = (node.children ?? [])[0];
-            const autoBroadcastOff = ctx.app?.config?.['spark.sql.autoBroadcastJoinThreshold']?.trim() === '-1';
+            // A threshold below the broadcast cannot have admitted it, so a hint forced it.
+            const threshold = effectiveBroadcastThreshold(ctx.app);
+            const autoBroadcastOff = threshold != null && threshold < 0;
+            const hintForced = autoBroadcastOff || (threshold != null && m.value > threshold);
+            const broadcastThreshold                     = autoBroadcastOff ? 'disabled' : hintForced ? 'notLimiting' : 'limits';
             out.push({
               type: 'overBroadcast', executionId: sqlExec.id,
               stageIds: unionStageIds(child ? [child] : [], fallbackStageIds),
               // resolvePlanTree always sets id; safe downstream of it.
               planNodeIds: [node.id ].filter(Boolean),
-              impactBand: 'warning', metric: 'broadcastBytes', value: m.value,
+              impactBand: 'warning', metric: 'broadcastBytes', value: m.value, broadcastThreshold,
               recommendation: autoBroadcastOff
                 ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: automatic broadcast is already disabled, so remove the broadcast() hint that forced it.`
-                : `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
-              remediation: autoBroadcastOff ? [] : [decreaseConf('spark.sql.autoBroadcastJoinThreshold')],
+                : hintForced
+                  ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: spark.sql.autoBroadcastJoinThreshold (${formatBytes(threshold )}) is below it, so a broadcast() hint forced it: remove the hint.`
+                  : `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
+              remediation: hintForced ? [] : [decreaseConf('spark.sql.autoBroadcastJoinThreshold')],
             });
           }
         }

package/vendor-core/docs-content/detection/chrn.md

@@ -3,6 +3,8 @@
 Executors are stood up and torn down again before they can do useful work:
 re-provisioning churn rather than normal scale-down. Raise
 `spark.dynamicAllocation.executorIdleTimeout`, or widen the
-`minExecutors`/`maxExecutors` bounds to reduce flapping. Self-flags a
+`minExecutors`/`maxExecutors` bounds to reduce flapping. With dynamic
+allocation off (`evidence.dynamicAllocation` is `off`) none of those applies:
+the churn is executor loss or preemption, and `remediation` is empty. Self-flags a
 confidence that scales with how far the short-lived-executor share sits
 past the threshold.

package/vendor-core/docs-content/detection/cold.md

@@ -3,4 +3,5 @@
 The first stage waited more than 30 s for an executor. Keep a warm pool of
 executors, or, with dynamic allocation, raise
 `spark.dynamicAllocation.minExecutors`/`initialExecutors` so the app doesn't
-scale up from zero.
+scale up from zero. With dynamic allocation off (`evidence.dynamicAllocation`
+is `off`) no executor-count property applies and `remediation` is empty.

package/vendor-core/docs-content/detection/plan.md

@@ -11,7 +11,14 @@ this tag:
   under 3 MB. Compact upstream output, or coalesce before writing.
 - Under-broadcast: the smaller side of a Sort Merge Join looks well under
   the broadcast threshold; consider a `broadcast()` hint or raising
-  `spark.sql.autoBroadcastJoinThreshold`.
+  `spark.sql.autoBroadcastJoinThreshold`. When the effective threshold
+  (logged, else Spark's 10 MiB) already admits the smaller side
+  (`evidence.broadcastThreshold` is `notLimiting`), the threshold is not what
+  stopped the broadcast, so `remediation` is empty and the advice is a hint or
+  table statistics.
 - Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a
   misapplied broadcast hint or a misconfigured
-  `spark.sql.autoBroadcastJoinThreshold`.
+  `spark.sql.autoBroadcastJoinThreshold`. When the effective threshold is
+  below the broadcast or auto-broadcast is disabled (`evidence.broadcastThreshold`
+  is `notLimiting` or `disabled`), a hint forced it: remove the hint, and
+  `remediation` is empty.

package/vendor-core/docs-content/detection/shfl.md

@@ -1,5 +1,11 @@
 ### `SHFL`: Shuffle I/O {#shfl}
 
 Tasks move a large amount of intermediate data between stages. Raise
-`spark.sql.shuffle.partitions`, or add a broadcast join. Only flagged on
-stages that take at least 0.5% of the run.
+`spark.sql.shuffle.partitions`, or add a broadcast join. The partition-count
+advice follows the effective conf (`evidence.partitions`): `raise` when the
+property limits the stage, `sufficient` when the tasks already read a good
+size each, `aqeCoalesced` when AQE merged the partitions (lower
+`spark.sql.adaptive.advisoryPartitionSizeInBytes`), and `ownPartitioning` when
+the property is already high enough (the stage's own `repartition(n)` or RDD
+parallelism limits it). Only flagged on stages that take at least 0.5% of the
+run.

package/vendor-core/docs-content/detection/skew.md

@@ -1,8 +1,14 @@
 ### `SKEW`: Task skew {#skew}
 
 A small number of tasks take much longer than their peers in the same
-stage. For join-driven skew, enable AQE skew-join handling
-(`spark.sql.adaptive.skewJoin.enabled`); otherwise salt the key or
-repartition on a better key. Flagged when P95 task time (the longest task,
-on a stage with fewer than 20 tasks) exceeds 3x the median and the
-recoverable tail is at least 0.5% of the run.
+stage. The fix depends on what the stage reads, which the finding's
+`evidence.origin` records. A stage that reads a shuffle feeding a join
+(`shuffleJoin`) gets AQE skew-join handling
+(`spark.sql.adaptive.skewJoin.enabled`), unless the run's effective conf
+already has it; otherwise salt the key or repartition on a better key. A
+stage that reads files with uneven sizes (`inputScan`) gets compaction of
+small files or a lower `spark.sql.files.maxPartitionBytes`. Any other stage
+(`other`) gets the salting advice and no conf. Flagged when P95 task time
+(the longest task, on a stage with fewer than 20 tasks) exceeds 3x the median
+and the recoverable tail is at least 0.5% of the run. The median is the
+textbook one: on an even task count, the mean of the two middle values.

package/vendor-core/docs-content/detection/slow.md

@@ -1,6 +1,10 @@
 ### `SLOW`: Stage slowness {#slow}
 
 A stage ran for 15 minutes or more and no slow host was flagged on it. It
-can appear alongside other findings on the same stage. Often a partition-count problem: raise parallelism via
+can appear alongside other findings on the same stage. On a stage that reads a
+shuffle, often a partition-count problem: raise parallelism via
 `spark.sql.shuffle.partitions` or `spark.default.parallelism`, or check for a
-large per-task data volume driving heavy shuffle and spill.
+large per-task data volume driving heavy shuffle and spill. On a stage that
+reads input files and no shuffle, check input file sizes and lower
+`spark.sql.files.maxPartitionBytes`. `evidence.reads` says which case
+applied (`shuffle`, `input` or `other`).

package/vendor-core/docs-content/detection/spill.md

@@ -4,4 +4,7 @@ Tasks are writing data out of memory, which slows execution. Two spill
 patterns get flagged differently: skew spill, where a few heavy tasks spill
 while most don't (rebalance partitioning), and volume spill, where most
 tasks spill because the data genuinely exceeds available memory (add
-partitions or executor memory). Only flagged on stages that take at least 0.5% of the run.
+partitions or executor memory). `spark.sql.shuffle.partitions` is suggested
+only for a stage that reads a shuffle; `evidence.reads` says what the stage
+reads (`shuffle`, `input` or `other`). Only flagged on stages that take at
+least 0.5% of the run.

package/vendor-core/docs-content/detection/strag.md

@@ -1,6 +1,8 @@
 ### `STRAG`: Straggler tasks {#strag}
 
 A few tasks run much slower than the rest of their stage. Rule out a GC
-pause or a slow shuffle fetch before assuming a hardware issue; if a skewed
-key is the real cause, that's a candidate for AQE's skew-join handling. Only
-flagged on stages that take at least 0.5% of the run.
+pause or a slow shuffle fetch before assuming a hardware issue. If uneven
+data is the cause, the advice is the one a skew finding gives for the same
+stage (`evidence.origin`, see `SKEW`): AQE skew-join handling for a shuffle
+feeding a join, file sizes for a scan, salting otherwise. Only flagged on
+stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/tiny.md

@@ -1,5 +1,6 @@
 ### `TINY`: Tiny tasks {#tiny}
 
 Many very short tasks add scheduling overhead out of proportion to the work
-each one does. Repartition to fewer, larger tasks. Only flagged on stages that
-take at least 0.5% of the run.
+each one does. Repartition to fewer, larger tasks; a stage that reads a shuffle
+(`evidence.reads` is `shuffle`) can also lower `spark.sql.shuffle.partitions`.
+Only flagged on stages that take at least 0.5% of the run.