sparkforensics-server 0.6.0 → 0.8.1

package/vendor-core/detectors.js

@@ -1,33 +1,38 @@
-import { pathBasename, formatBytes, IMPACT_BAND_ORDER, MS_PER_CORE_HOUR } from './format-utils.js';
+import { pathBasename, formatBytes, formatDuration, IMPACT_BAND_ORDER, MS_PER_CORE_HOUR } from './format-utils.js';
 import { medianOfSorted } from './median.js';
 import { shareLabel } from './finding-presentation.js';
-import { scanRelationId } from './plan-summary.js';
+import { scanRelationId, parseJoinType } from './plan-summary.js';
 import { allocatedCoreMs, computeAllocation } from './allocation.js';
-import { computePeakConcurrentCores, computePeakConcurrentExecutorCount } from './core-count.js';
+import { executorHeapPeaks } from './executor-peaks.js';
+import { computePeakConcurrentCores } from './core-count.js';
 import { walkPlanTree } from './plan-tree-walk.js';
+import { diagnoseJoinSkew, isSkewJoinNode, planShowsSkewSplit } from './aqe-skew.js';
+import { isBatchEvalPythonNode } from './python-stage.js';
 import { computeCoreLocalityRatio } from './core-locality-ratio.js';
+import { STRAGGLER_FACTOR } from './stage-quantiles.js';
 import { tailRecoveryMs, tailRemovedWorkMs, stragglerFixLongestTaskMs,                } from './occupancy.js';
 import { IMPACT_FLOOR_PCT_WARN, IMPACT_FLOOR_PCT_CRIT, appDurationMs } from './impact-band.js';
 import {
   BROADCAST_BANDWIDTH_BPS, EXECUTOR_STARTUP_OVERHEAD_MS, FILE_OPEN_OVERHEAD_MS, IDEAL_BYTES_PER_PARTITION_TASK,
   NETWORK_FETCH_PENALTY_MS, RE_READ_THROUGHPUT_BPS, SHUFFLE_THROUGHPUT_BPS, SPILL_IO_THROUGHPUT_BPS, TAIL_CLAIM,
   TASK_SCHEDULING_OVERHEAD_MS, costOnly, fetchWaitWallClockMs, measuredTaskOverhead, multiStageImpact, noWasteModel,
-  retryWallClockMs, singleStageImpact, stageIoParallelism, stageMappableWasteOrCostOnly, tasksMostlyIdle,
+  ioModelAssumption, retryWallClockMs, singleStageImpact, stageIoParallelism, stageMappableWasteOrCostOnly, tasksMostlyIdle, withAssumptions,
                    
 } from './impact-model.js';
 import { isExchangeNode, isBroadcastExchangeNode } from './plan-node-detail.js';
+import { outputRowsOf, ownOutputRows, parseNestedLoopJoin } from './nested-loop-join.js';
 import { totalExecutorCpuMs } from './run-totals.js';
 import { DUPLICATE_SUBTREE_DIFFERING_NOTE, duplicateSubtreeDetail, SLOW_HOST_DIMENSION_LABEL } from './finding-generic-recommendation.js';
 import { stageIdsForSqlExec } from './sql-stages.js';
 import { cyrb53 } from './string-hash.js';
 import { codeFix, decreaseConf, increaseConf, setConf } from './remediation.js';
+import { effectiveSparkConf, executorOverheadSettings, overlayModifiedConfigs, parseSparkBytes } from './spark-conf.js';
 import { MAX_FAILURE_GROUPS, describeTaskFailure,                       } from './task-failure.js';
-                                                                                               
-                                                                                                                                                                                     
+                                                                                                                
+                                                                                                                                                                                                             
 
 const MB = 1024 * 1024;
 const GB = 1024 * MB;
-const TB = 1024 * GB;
 
 // Labels a byte threshold in this file's binary units, so a 1 GiB default reads '1 GB'.
 function binaryThresholdLabel(bytes        )         {
@@ -45,6 +50,9 @@ function binaryThresholdLabel(bytes        )         {
                                                                                       
                                 
                                                                
+                                                                                                   
+                                                               
+                                                  
                                                                           
  
                                                                   
@@ -93,6 +101,7 @@ function binaryThresholdLabel(bytes        )         {
                              
                                  
                                
+                                    
                            
                                     
                              
@@ -106,6 +115,8 @@ function binaryThresholdLabel(bytes        )         {
                                   
              
                              
+                                                                                      
+                                           
  
 
                            
@@ -157,16 +168,33 @@ function binaryThresholdLabel(bytes        )         {
                                                    
                                  
                                     
-                                                 
+                                                                                                               
                                                                                                 
                                                                                 
                       
+                                                                                                   
+                                                                                 
+                                 
  
 
 // auditConfig(app) calls every config-scope detect({ app }) with whatever appModel.app is
 // (SparkAppInfo | null), independent of DetectorCtx.
                                        
                           
+                                                                                                      
+                                                      
+                                      
+ 
+
+// True when any stage ran outside a SQL execution: RDD API work, which spark.serializer governs.
+export function hasStageOutsideSql(stages                                )          {
+  if (stages == null) return false;
+  for (const stage of stages.values()) if (stage.sqlExecutionId == null) return true;
+  return false;
+}
+
+                                 
+                                                                                        
  
 
                            
@@ -177,13 +205,13 @@ function binaryThresholdLabel(bytes        )         {
                         
  
 
-// SpillPressureDetector (5a) + SpillSkewDetector (5b).
+// Spill magnitude: absolute-volume tiers first, then ratio-plus-floor skew tiers.
 function computeSpillMagnitude(
   stage                                                                                                    ,
   t                 ,
 )                                                     {
   const { taskCount, spillDiskMax = 0, spillMemMax = 0, spillDiskP50 = 0, spillMemP50 = 0 } = stage;
-  // 5a: absolute volume, branching on task count.
+  // Absolute volume, branching on task count.
   if (taskCount === 1) {
     if (spillDiskMax >= t.singleTaskDiskGiB * GB || spillMemMax >= t.singleTaskMemGiB * GB) return { magnitude: 'severe' };
   } else {
@@ -191,7 +219,7 @@ function computeSpillMagnitude(
     if (spillDiskMax >= t.highDiskGiB * GB || perTaskDisk >= t.highTaskDiskMB * MB || spillMemMax >= t.highMemGiB * GB) return { magnitude: 'high' };
     if (spillDiskMax >= t.medDiskMB * MB || spillMemMax >= t.medMemGiB * GB) return { magnitude: 'medium' };
   }
-  // 5b: ratio+floor skew (requires enough tasks).
+  // Ratio plus floor skew (requires enough tasks).
   if (taskCount >= t.skewMinTasks) {
     if (spillDiskP50 > 0 && spillDiskMax / spillDiskP50 > t.skewRatio && spillDiskMax >= t.skewDiskFloorMB * MB) return { magnitude: 'high' };
     if (spillMemP50 > 0 && spillMemMax / spillMemP50 > t.skewRatio && spillMemMax >= t.skewMemFloorMB * MB) return { magnitude: 'medium' };
@@ -215,7 +243,7 @@ export function unionStageIds(nodes            , fallback          )           {
 
 // Bottom-up shape computation for duplicate-subtree detection and the
 // cachingOpportunity composite detector. `size` is the subtree node count; the default
-// fingerprint encodes operator name + sorted metric NAMES (never values, per spec) + a digest
+// fingerprint encodes operator name + sorted metric NAMES (never values) + a digest
 // of each child's fingerprint, so two subtrees with the same shape but different values still
 // collide. Digesting children keeps each fingerprint O(own size): embedding the full child
 // strings made every node carry its whole subtree, O(n x depth) text on deep real plans.
@@ -225,6 +253,13 @@ export function unionStageIds(nodes            , fallback          )           {
 // without descendant scan detail entering the comparison.
                                                           
 
+// The metric names a node's fingerprint is built from. Executor-side values are left out: whether
+// one resolves depends on which tasks reported it, and a fingerprint (and the duplicatePlanSubtree
+// and cachingOpportunity ids derived from it) must not shift when a run reports more of them.
+function fingerprintMetricNames(metrics                     )         {
+  return (metrics ?? []).filter((m) => !m.executorSide).map((m) => m.name).sort().join(',');
+}
+
 export function computePlanShapes(
   root          ,
   opts                                                                       = {},
@@ -249,7 +284,7 @@ export function computePlanShapes(
     const realMetrics = writeHalf ? writeHalf.metrics : node.metrics;
     const childShapes = realChildren.map((c) => visit(c, false));
     const size = 1 + childShapes.reduce((sum, c) => sum + c.size, 0);
-    const metricNames = (realMetrics ?? []).map((m) => m.name).sort().join(',');
+    const metricNames = fingerprintMetricNames(realMetrics);
     const childFingerprints = childShapes.map((c) => cyrb53(c.fingerprint)).join(',');
     const fingerprint = isRoot && includeDetail
       ? `${node.name}[${metricNames}]<${normalize(node.detail ?? '')}>{${childFingerprints}}`
@@ -274,8 +309,7 @@ export function normalizeDetail(detail        )         {
     .replace(/\bBuild(Left|Right)\b/g, 'BuildSide');
   // The lookbehind only skips starts inside an identifier, which can't match unless the
   // identifier's own start already did: same output, without re-scanning each identifier from
-  // every one of its letters (O(length^2) on the 6.8 KB average join detail of the largest real
-  // log; 64ms -> 40ms per analyze()).
+  // every one of its letters (O(length^2) on long join details).
   s = s.replace(/(?<![A-Za-z_])([A-Za-z_][\w.]*)\s*=\s*([A-Za-z_][\w.]*)/g, (_m, l, r) => {
     const [a, b] = [l, r].sort();
     return `${a} = ${b}`;
@@ -331,7 +365,7 @@ export function findCompositeCandidates(root          )                       {
     const childResults = (node.children ?? []).map(visit);
     path.pop();
 
-    const metricNames = (node.metrics ?? []).map((m) => m.name).sort().join(',');
+    const metricNames = fingerprintMetricNames(node.metrics);
     // Child digests, as in computePlanShapes: deterministic, so still comparable across executions.
     const childFingerprints = childResults.map((r) => cyrb53(r.fingerprint)).join(',');
     const fingerprint = `${node.name}[${metricNames}]{${childFingerprints}}`;
@@ -398,11 +432,8 @@ export function findDuplicateSubtrees(
   { minSubtreeSize, minOccurrences }                                                    ,
 )                          {
   const { shapeOf, allNodes } = computePlanShapes(root);
-  // Defensive, not load-bearing: computePlanShapes's visit() already skips
-  // straight through a read node to its write half's real children, so no
-  // write-half node is ever pushed into allNodes in the first place, this
-  // filter can structurally never exclude anything. Kept in case that
-  // invariant ever changes upstream.
+  // Guard only: computePlanShapes steps through a read node to its write half's children, so no
+  // write-half node reaches allNodes.
   const eligible = allNodes.filter(n => shapeOf.get(n) .size >= minSubtreeSize && n.exchangeRole !== 'write');
 
   const groups = new Map                    ();
@@ -446,7 +477,7 @@ export function findDuplicateSubtrees(
 // scanned table, literals). The fingerprint ignores detail, so occurrences can be same-shaped
 // branches over different data; only identical ones are repeated work that computing once would
 // save. AQE query-stage numbers (`ShuffleQueryStage 718` vs `720`) name the same computation's
-// runtime stage and are ignored too. On the 14 real logs, 270 of 546 groups differed beyond that.
+// runtime stage and are ignored too. Calibration: threshold-tables.md.
 function occurrencesHaveIdenticalDetails(occurrences            )          {
   const detailsOf = (root          )           => {
     const out           = [];
@@ -470,11 +501,10 @@ function occurrencesHaveIdenticalDetails(occurrences            )          {
 
 // Stages that run nothing but the duplicated occurrences' operators: every operator attributed to
 // the stage is inside `nodes`. A stage shared with operators outside them (the join consuming the
-// subtree, the other join side) does other work too, so its time isn't the subtree's to claim;
-// counting it also let sibling groups claim the same stage twice. A WholeStageCodegen wrapper and
+// subtree, the other join side) does other work too, so its time isn't the subtree's to claim, and
+// counting it would let sibling groups claim the same stage twice. A WholeStageCodegen wrapper and
 // an Exchange's write half aren't other work (the fused pipeline around an operator, the shuffle
-// write of its output), so they're left out of both counts; on the 14 real logs they were the
-// only outside node on 426 stages.
+// write of its output), so they're left out of both counts.
 function countsTowardStage(node          )          {
   return node.exchangeRole !== 'write' && !node.name.startsWith('WholeStageCodegen');
 }
@@ -517,40 +547,119 @@ const FILES_READ_BYTES = 'size of files read';
 const FILES_WRITTEN_COUNT = 'number of written files';
 const FILES_WRITTEN_BYTES = 'written output';
 
-// Byte size of a join-side subtree for broadcast sizing. Stops
-// descending at a node with a "data size" metric: that value already aggregates everything
-// beneath it, so summing further would double-count. Only recurses when no such metric.
-function sumBoundarySize(node          )         {
-  const m = (node.metrics ?? []).find(x => x.name === 'data size');
-  if (m) return m.value;
-  let sum = 0;
-  for (const c of (node.children ?? [])) sum += sumBoundarySize(c);
-  return sum;
+// A join side's shuffle: the Exchange node whose "data size" is that side's runtime size. AQE
+// decides broadcast from the same metric (ShuffleExchangeExec.runtimeStatistics, apache/spark
+// v3.5.0). Only operators that pass rows through unchanged are looked through, so a side that is
+// another join's output, an aggregate, or a reused exchange has no size of its own and yields null.
+const SHUFFLE_PASS_THROUGH = /^(Sort|InputAdapter|WholeStageCodegen|ShuffleQueryStage|AQEShuffleRead|CustomShuffleReader)\b/;
+
+function joinSideShuffle(node          )                  {
+  if (node.name === 'ReusedExchange') return null;
+  const dataSize = (node.metrics ?? []).some((m) => m.name === 'data size');
+  if (isExchangeNode(node) && !isBroadcastExchangeNode(node.name) && dataSize) return node;
+  const children = node.children ?? [];
+  if (children.length !== 1 || !(isExchangeNode(node) || SHUFFLE_PASS_THROUGH.test(node.name))) return null;
+  return joinSideShuffle(children[0]);
 }
 
-// Nodes that fed sumBoundarySize's total: mirrors its recursion exactly so implicated stageIds
-// line up with the size actually compared.
-function boundarySizeContributors(node          )             {
-  const m = (node.metrics ?? []).find((x) => x.name === 'data size');
-  if (m) return [node];
-  const out             = [];
-  for (const c of (node.children ?? [])) out.push(...boundarySizeContributors(c));
-  return out;
+function shuffleBytes(exchange          )         {
+  return (exchange.metrics ?? []).find((m) => m.name === 'data size')?.value ?? 0;
+}
+
+// The join types a side can be the broadcast build side of, per canBuildBroadcastLeft/Right in
+// JoinSelectionHelper (apache/spark v3.5.0, v4.0.0 adds LeftSingle on the right). A full outer
+// join builds from neither side, and an unrecognized type is not guessed at.
+const BROADCAST_BUILD_LEFT                      = new Set(['Inner', 'Cross', 'RightOuter']);
+const BROADCAST_BUILD_RIGHT                      = new Set(['Inner', 'Cross', 'LeftOuter', 'LeftSingle', 'LeftSemi', 'LeftAnti', 'ExistenceJoin']);
+
+                              
+                   
+                              
+                  
+                   
+ 
+
+// The smaller of the sides this join's type can broadcast. Both sides need a measurable size of
+// their own, so the candidate is never compared with an unknown partner.
+function broadcastCandidate(join          )                            {
+  const joinType = parseJoinType(join.detail ?? '');
+  if (joinType == null || (join.children ?? []).length !== 2) return null;
+  const left = joinSideShuffle(join.children[0]);
+  const right = joinSideShuffle(join.children[1]);
+  if (!left || !right) return null;
+  const options                       = [];
+  if (BROADCAST_BUILD_LEFT.has(joinType)) options.push({ joinType, buildSide: 'left', build: left, stream: right });
+  if (BROADCAST_BUILD_RIGHT.has(joinType)) options.push({ joinType, buildSide: 'right', build: right, stream: left });
+  return options.reduce                           ((best, o) => (best == null || shuffleBytes(o.build) < shuffleBytes(best.build) ? o : best), null);
 }
 
 // Max/median deviation ratio over a list of {key, value}; returns the
-// exceeding entry's key + ratio, or null when < 3 samples or median 0.
+// exceeding entry's key + ratio, or null when < 3 samples or median 0. `qualifies` narrows the
+// entries the maximum is taken from (the median still covers all of them).
 function maxMedianRatio(
   samples                                  ,
+  qualifies                                                      = () => true,
 )                                                       {
   if (samples.length < 3) return null;
   const vals = samples.map(s => s.value).sort((a, b) => a - b);
   const median = medianOfSorted(vals);
   if (median <= 0) return null;
-  const top = samples.reduce((a, b) => (b.value > a.value ? b : a));
+  const pool = samples.filter(qualifies);
+  if (pool.length === 0) return null;
+  const top = pool.reduce((a, b) => (b.value > a.value ? b : a));
   return { key: top.key, ratio: top.value / median, value: top.value };
 }
 
+// z a lone executor's mean must clear to stand out from `executors` chance draws at a family-wise
+// false-flag rate of `alpha`. A normal tail obeys P(Z > z) <= exp(-z^2 / 2) / 2, so a
+// Bonferroni share of alpha / executors gives z = sqrt(2 ln(executors / (2 alpha))): the cutoff
+// grows with the number of executors the largest mean was picked from.
+function chanceOutlierZ(executors        , alpha        )         {
+  return Math.sqrt(2 * Math.log(executors / (2 * alpha)));
+}
+
+// How far each executor's task durations sit above the stage's median executor, in standard errors,
+// on the log scale: (executor mean - median of executor means) / (stage task sd / sqrt(executor
+// tasks)). The stage sd is rebuilt from the per-executor count, log sum and M2: within-executor
+// squared deviations plus each executor mean's distance from the grand mean. null when the stage
+// has under two tasks, no spread, or any executor lacks finite log stats, so an unknown spread
+// never flags an executor.
+function executorDurationZ(execs                        )                             {
+  const active = execs.filter(e => e.taskCount > 0);
+  let tasks = 0, logTotal = 0;
+  for (const e of active) {
+    if (!Number.isFinite(e.logDurationSum) || !Number.isFinite(e.logDurationM2)) return null;
+    tasks += e.taskCount;
+    logTotal += e.logDurationSum          ;
+  }
+  if (tasks < 2) return null;
+  const grandMean = logTotal / tasks;
+  let squares = 0;
+  const means = active.map(e => (e.logDurationSum          ) / e.taskCount);
+  active.forEach((e, i) => { squares += (e.logDurationM2          ) + e.taskCount * (means[i] - grandMean) ** 2; });
+  const sd = Math.sqrt(squares / (tasks - 1));
+  if (!(sd > 0) || !Number.isFinite(sd)) return null;
+  const median = medianOfSorted([...means].sort((a, b) => a - b));
+  return new Map(active.map((e, i) => [e.executorId, (means[i] - median) / (sd / Math.sqrt(e.taskCount))]));
+}
+
+// slowHost's taskTime dimension picks the largest of N per-executor mean task times: the more
+// executors, the likelier the largest reaches a given ratio to the median by chance alone, whatever
+// the hardware. The band comes from the modeled saving, so the finding states how weak the signal is.
+// No `host` here: it is one of slowHost's id discriminators, so setting it would change the id.
+function executorTaskTimeCaveat(
+  executorId        , ratio        , executorCount        , taskCount        ,
+)                                                                                             {
+  return {
+    confidence: 'low',
+    validationRequired:
+      `One of ${executorCount} executors: the largest of ${executorCount} per-executor mean task times reaches ${Math.round(ratio * 10) / 10}x the median by chance ` +
+      `when task durations vary widely or each executor ran few tasks (${taskCount} tasks over ${executorCount} executors here). ` +
+      `Compare per-task times on executor ${executorId} in the Spark UI before acting.`,
+    executorCount, taskCount,
+  };
+}
+
                                        
                      
                   
@@ -595,7 +704,6 @@ export function computeSkewRatio(
 // ratio on a few ms is noise in an hours-long run but real in a seconds-long one; a fixed-ms
 // floor can't scale. Used by skew/straggler, gated on the same occupancy-clipped tail claim their
 // estimate() displays as savings (tailClaimImpact).
-// NOT SOURCED: floor percentages are our own noise floor, unvalidated.
 // Unknown app timing never suppresses a finding; it just skips the floor gate.
 function meetsRuntimeFloor(wasteMs        , runMs               , floorPct        )          {
   return runMs == null || wasteMs >= runMs * floorPct;
@@ -657,26 +765,90 @@ function tailClaimFloorMs(claim           , stageId        , ctx             )
   return tailClaimImpact(claim, stageId, ctx.impact).wallClock?.high ?? claim.wasteMs;
 }
 
-// Spark's default for the properties a detector's fix reads or suggests, for a run that did not log
-// them. Undefined when the property is not modeled or its default depends on a version the log did
-// not record. A property is modeled only where its default has made a suggestion a no-op; every
-// other property a detector suggests defaults off, or to a value other than the suggested one.
-function versionDefault(app                    , key        )                     {
-  if (key === 'spark.sql.shuffle.partitions') return '200';
-  if (key === 'spark.sql.autoBroadcastJoinThreshold') return String(10 * MB);
-  const version = /^(\d+)\.(\d+)/.exec(app?.sparkVersion ?? '');
-  if (version == null) return undefined;
-  const [major, minor] = [Number(version[1]), Number(version[2])];
-  if (key === 'spark.sql.adaptive.enabled') return major > 3 || (major === 3 && minor >= 2) ? 'true' : 'false';
-  if (key === 'spark.sql.adaptive.skewJoin.enabled' || key === 'spark.sql.adaptive.coalescePartitions.enabled') {
-    return major >= 3 ? 'true' : undefined;
-  }
-  return undefined;
+// skew's duration gate: the task-time ratio over `ratioWarn` whose occupancy-clipped tail claim
+// (the claim estimate() reports as savings, so the gate agrees with it) is at least `floorPctWarn`
+// of the run. Null when it does not hold. straggler runs it too, to report the tails skew's gate
+// admits whose cause is not data.
+function skewGate(
+  stage               , ctx             , ratioWarn        , minTasksForP95        , floorPctWarn        ,
+)                                                                {
+  const result = computeSkewRatio(stage, minTasksForP95);
+  if (result === null || result.ratio <= ratioWarn) return null;
+  const floorWasteMs = tailClaimFloorMs(skewTailClaim(stage, result.metric === 'P95/median'), stage.id, ctx);
+  return meetsRuntimeFloor(floorWasteMs, appDurationMs(ctx.app), floorPctWarn) ? result : null;
 }
 
-// The run's effective value of a property: the logged one, else versionDefault's.
+// Why a stage's slow tail is slow, from attributeTail (stage-quantiles.ts). 'data' when data volume
+// accounts for at least `dataShareMin` of the tail's extra time, else the largest of GC, fetch wait,
+// one host and 'unexplained'. 'unattributed' (no data volume to compare, or no tail attribution)
+// means only the duration is known, so straggler reports it and skew does not.
+                       
+                   
+                                                                                         
+                   
+                                      
+ 
+
+function tailVerdict(stage                                        , dataShareMin        )              {
+  const a = stage.tailAttribution ?? null;
+  if (a === null || !(a.excessMs > 0)) return { cause: 'unattributed', sharePct: 0, attribution: a };
+  const pct = (ms        ) => Math.round((ms / a.excessMs) * 100);
+  if (a.dataMs / a.excessMs >= dataShareMin) return { cause: 'data', sharePct: pct(a.dataMs), attribution: a };
+  const rest = Math.max(0, a.excessMs - a.dataMs - a.gcMs - a.fetchWaitMs - a.hostMs);
+  const causes                        = [['gc', a.gcMs], ['fetchWait', a.fetchWaitMs], ['host', a.hostMs],
+    [a.dataRatio === null ? 'unattributed' : 'unexplained', rest]];
+  const [cause, ms] = causes.reduce((best, c) => (c[1] > best[1] ? c : best));
+  return { cause, sharePct: cause === 'unattributed' ? 0 : pct(ms), attribution: a };
+}
+
+// The run's effective value of a property: the logged one, else Spark's default for the run's
+// version (spark-conf.ts). A detector scoped to one SQL execution passes that execution's app (see
+// queryApp) so the settings it ran with count.
 function effectiveConf(app                    , key        )                     {
-  return (app?.config?.[key] ?? versionDefault(app, key))?.trim();
+  return effectiveSparkConf({ sparkVersion: app?.sparkVersion, properties: app?.config }, key)?.value;
+}
+
+// The factor and minimum Spark applies to size the executor overhead when spark.executor.memoryOverhead
+// is unset (spark-conf.ts's executorOverheadSettings); `fallback` is the detector's default pair.
+function defaultOverheadSettings(app                    , fallback                                       )                                    {
+  return executorOverheadSettings({ sparkVersion: app?.sparkVersion, properties: app?.config }, fallback);
+}
+
+const queryAppCache = new WeakMap                                                            ();
+
+// The app as one SQL execution saw it: its modifiedConfigs applied over the logged properties, so
+// effectiveConf reads what `spark.conf.set` changed. The app itself when the execution modified
+// nothing.
+function queryApp(app                    , modified                                    )                     {
+  if (modified == null) return app;
+  const cached = queryAppCache.get(modified);
+  if (cached?.base === app) return cached.derived;
+  const config = overlayModifiedConfigs(app?.config, modified);
+  if (config === app?.config) return app;
+  const derived              = { ...app, config };
+  queryAppCache.set(modified, { base: app, derived });
+  return derived;
+}
+
+// The app as the stage's SQL execution saw it; the run's app for a stage outside any SQL execution.
+function stageApp(ctx             , stage                                       )                     {
+  return queryApp(ctx.app, stage.sqlExecutionId != null ? ctx.sql.get(stage.sqlExecutionId)?.modifiedConfigs : undefined);
+}
+
+// When Spark relaunches a slow task under the run's effective speculation settings: once the
+// quantile of the stage's tasks has finished, a task running over the multiplier times the median
+// of the finished ones gets a copy (TaskSetManager.checkSpeculatableTasks). Null when either
+// setting is neither logged nor a default the run's Spark version fixes.
+function speculationRule(app                    )                {
+  const multiplier = Number.parseFloat(effectiveConf(app, 'spark.speculation.multiplier') ?? '');
+  const quantile = Number.parseFloat(effectiveConf(app, 'spark.speculation.quantile') ?? '');
+  if (!Number.isFinite(multiplier) || !Number.isFinite(quantile)) return null;
+  return `a task running over ${multiplier}x the median is relaunched once ${Math.round(quantile * 100)}% of the stage's tasks have finished`;
+}
+
+function speculationRuleNote(app                    )         {
+  const rule = speculationRule(app);
+  return rule == null ? '' : ` (with this run's settings, ${rule})`;
 }
 
 // A setting is only a fix when the run's effective conf doesn't already have it: a run that set it,
@@ -699,6 +871,34 @@ function switchFix(on         , key        , suggested                  , recomm
   return on ? { text: alreadyOn, remediation: [] } : { text: recommend, remediation: [setConf(key, suggested)] };
 }
 
+// Whether memoryUtilization can judge the executor heap: a measured peak and a known executor size.
+// Without both, the stage-level low-GC note is the only memory-sizing signal there is.
+function heapJudgementAvailable(ctx                                                       )          {
+  return executorHeapPeaks(ctx).size > 0 && (ctx.app?.resources?.executor?.memoryMB ?? 0) > 0;
+}
+
+const LOG_STAGE_EXECUTOR_METRICS_KEY = 'spark.eventLog.logStageExecutorMetrics';
+const PYTHON_UDF_ARROW_KEY = 'spark.sql.execution.pythonUDF.arrow.enabled';
+// The executor-side metrics of the Python evaluator operators. Spark's PythonSQLMetrics trait adds
+// them in 3.4.0, the release that also adds the Arrow-optimized UDF property, so an older log never
+// reports them and the detector skips it.
+const PYTHON_DATA_SENT = 'data sent to Python workers';
+const PYTHON_DATA_RETURNED = 'data returned from Python workers';
+
+// How to cut the cost of a row-at-a-time Python UDF: the Arrow-optimized UDF property while the
+// session lacks it, else what is left once it is on (a UDF that opted out with useArrow=False, takes no
+// arguments, or was created before the property was set: all of them still run as BatchEvalPython).
+function pythonUdfArrowFix(app                    )                                               {
+  return switchFix(
+    loggedAs(app, PYTHON_UDF_ARROW_KEY, true), PYTHON_UDF_ARROW_KEY, true,
+    `set ${PYTHON_UDF_ARROW_KEY}=true to ship rows in Arrow batches, or rewrite the UDF as a pandas UDF.`,
+    `Arrow-optimized Python UDFs are already on for the session, so this UDF may have opted out with useArrow=False, take no arguments, or have been created before the property was set: check which, or rewrite it as a pandas UDF.`,
+  );
+}
+
+// A tail reading at least this many times the median task's data is not described as reading no more.
+const MORE_DATA_RATIO = 2;
+
 const SKEW_KEY_REMEDY = 'salt the key or repartition on a better key';
 
 // What a stage reads, by the dominant side: a stage that reads a shuffle and mostly input files is
@@ -717,24 +917,22 @@ function predatesAqeSkewJoin(app                    )          {
 // What a skewed stage reads, which decides whether AQE skew-join handling can act on it. It splits
 // skewed partitions on the shuffle-read side of a sort-merge or shuffled-hash join, so only a stage
 // that reads a shuffle, in a SQL execution whose plan has such a join, is 'shuffleJoin'. A plan does
-// not say which stage runs which join (a node's stageIds are empty in real logs), so the join is
-// matched per execution. 'inputScan' reads mostly files (its tasks differ in input size);
+// not say which stage runs which join, so the join is matched per execution. 'inputScan' reads mostly files (its tasks differ in input size);
 // 'other' is any stage the plan cannot tie to a join (an aggregation's shuffle, no plan, no input),
 // and any stage of a run on Spark before 3.0 (skewFix).
 function skewOrigin(stage               , ctx             , shuffleEvidence         )                                        {
   if (shuffleEvidence || (stage.shuffleReadBytes > 0 && stage.shuffleReadBytes >= stage.inputBytes)) {
     let hasJoin = false;
     const plan = stage.sqlExecutionId != null ? ctx.sql.get(stage.sqlExecutionId)?.planTree : null;
-    walkPlanTree(plan, (node) => { if (SKEW_JOIN_NODES.has(node.name)) hasJoin = true; });
+    walkPlanTree(plan, (node) => { if (isSkewJoinNode(node.name)) hasJoin = true; });
     return hasJoin ? 'shuffleJoin' : 'other';
   }
   return stage.inputBytes > 0 ? 'inputScan' : 'other';
 }
-const SKEW_JOIN_NODES = new Set(['SortMergeJoin', 'ShuffledHashJoin']);
 
 // The skew finding's fix for the stage's origin. Join-driven skew gets AQE skew-join handling,
 // unless the run's effective conf already has it; uneven input gets the file-size remedy.
-function skewFix(stage               , ctx             , shuffleEvidence = false)                                                                   {
+function skewFix(stage               , ctx             , shuffleEvidence = false, tail              )                                                                                          {
   const origin = skewOrigin(stage, ctx, shuffleEvidence);
   if (origin === 'inputScan') {
     return {
@@ -746,7 +944,22 @@ function skewFix(stage               , ctx             , shuffleEvidence = false
   if (origin === 'other') return { origin, text: SKEW_KEY_REMEDY, remediation: [codeFix(SKEW_KEY_REMEDY)] };
   // Before Spark 3.0 there is no AQE skew-join handling, so the stage gets the generic advice.
   if (predatesAqeSkewJoin(ctx.app)) return { origin: 'other', text: SKEW_KEY_REMEDY, remediation: [codeFix(SKEW_KEY_REMEDY)] };
-  return { origin, ...skewJoinFix(ctx.app) };
+  const app = stageApp(ctx, stage);
+  const fix = skewJoinFix(app);
+  // A tail the data volume explains (input plus shuffle read, in bytes or records) is data the
+  // diagnosis must not call even or cured from shuffle-read bytes alone.
+  const tailData = tail?.cause === 'data' ? { ratio: tail.attribution?.dataRatio ?? null } : undefined;
+  // With skew-join handling on, the final plan and conf say why it did or did not act on this stage.
+  const handlingOn = !loggedAs(app, 'spark.sql.adaptive.enabled', false) && loggedAs(app, 'spark.sql.adaptive.skewJoin.enabled', true);
+  const diagnosis = handlingOn && stage.sqlExecutionId != null
+    ? diagnoseJoinSkew({
+      plan: ctx.sql.get(stage.sqlExecutionId)?.planTree, stageId: stage.id,
+      readMax: stage.shuffleReadMax, readP50: stage.shuffleReadP50, stageShuffleReadBytes: stage.shuffleReadBytes,
+      ...(tailData ? { tailData } : {}),
+      conf: (key) => effectiveConf(app, key), sparkVersion: app?.sparkVersion, keyRemedy: SKEW_KEY_REMEDY,
+    })
+    : null;
+  return diagnosis == null ? { origin, ...fix } : { origin, aqeSkew: diagnosis.case, text: diagnosis.text, remediation: diagnosis.remediation };
 }
 
 function skewJoinFix(app                    )                                               {
@@ -780,7 +993,7 @@ function idleCapacityFix(app             , recommend        , alreadyOnLead
   const fix = dynamicAllocationFix(app, recommend, alreadyOnLead);
   // dynamicAllocationFix's remediation is empty exactly when dynamic allocation is already on.
   if (fix.remediation.length > 0) return { ...fix, remediation: [...fix.remediation, decreaseConf('spark.executor.instances')] };
-  const holdsFloor = Number.parseInt(app.config?.['spark.dynamicAllocation.minExecutors'] ?? '', 10) > 0;
+  const holdsFloor = Number.parseInt(effectiveConf(app, 'spark.dynamicAllocation.minExecutors') ?? '', 10) > 0;
   return holdsFloor
     ? { text: `${alreadyOnLead} by lowering spark.dynamicAllocation.maxExecutors and spark.dynamicAllocation.minExecutors`,
         remediation: [decreaseConf('spark.dynamicAllocation.maxExecutors'), decreaseConf('spark.dynamicAllocation.minExecutors')] }
@@ -794,31 +1007,129 @@ function effectiveShufflePartitions(app                    )                {
   return value != null && /^\d+$/.test(value) ? Number(value) : null;
 }
 
-// Whether partition-count advice fits a shuffle-reading stage, from the effective conf and the
-// stage's own task sizes. The partition count that brings each shuffle partition down to the ideal
-// size is per stage; the property is job-wide. 'sufficient': the stage's tasks are already at or
-// under the ideal size. 'aqeCoalesced': AQE merged the property's partitions into fewer tasks, so
-// the advisory partition size is the lever. 'ownPartitioning': the property already gives at least
-// the count needed, so the stage's own repartition(n) or RDD parallelism limits it. Otherwise the
-// property limits the stage: 'raise'.
-function shufflePartitionCase(stage               , app                    )                                                                          {
-  const count = effectiveShufflePartitions(app);
+// The partition count and property that sized a shuffle-reading stage's tasks. AQE only rewrites the
+// count of a stage inside a SQL execution. With coalescing on, `initialPartitionNum` replaces
+// spark.sql.shuffle.partitions as the starting count when it is set
+// (CoalesceShufflePartitions.coalesceShufflePartitions at v3.5.0), so it is the lever then.
+const INITIAL_PARTITION_NUM_KEY = 'spark.sql.adaptive.coalescePartitions.initialPartitionNum';
+
+                          
+                                                                                                        
+                       
+                                                                                        
+              
+                                                                                                       
+               
+                                                                  
+                    
+                                                                                                           
+                            
+                                                                                             
+                               
+ 
+
+function partitionBasis(stage               , ctx             )                 {
+  const app = stageApp(ctx, stage);
+  const aqe = stage.sqlExecutionId != null && loggedAs(app, 'spark.sql.adaptive.enabled', true);
+  const coalesce = aqe && loggedAs(app, 'spark.sql.adaptive.coalescePartitions.enabled', true);
+  const initial = coalesce ? effectiveConf(app, INITIAL_PARTITION_NUM_KEY) : undefined;
+  const plan = stage.sqlExecutionId != null ? ctx.sql.get(stage.sqlExecutionId)?.planTree : null;
+  const skewSplit = aqe ? planShowsSkewSplit(plan) : null;
+  const explicitRepartition = planHasExplicitRepartition(plan);
+  if (initial != null && /^\d+$/.test(initial)) return { count: Number(initial), key: INITIAL_PARTITION_NUM_KEY, aqe, coalesce, skewSplit, explicitRepartition };
+  return { count: effectiveShufflePartitions(app), key: 'spark.sql.shuffle.partitions', aqe, coalesce, skewSplit, explicitRepartition };
+}
+
+// Whether a plan holds an exchange the job asked for (`repartition(n)`, `repartition(col)` or a
+// rebalance hint), which is what makes a task count different from the configured one the stage's own.
+function planHasExplicitRepartition(plan                             )          {
+  let found = false;
+  walkPlanTree(plan, (node) => { if (/\b(REPARTITION_BY_NUM|REPARTITION_BY_COL|REBALANCE_PARTITIONS_BY_NONE|REBALANCE_PARTITIONS_BY_COL)\b/.test(node.detail ?? '')) found = true; });
+  return found;
+}
+
+// How the stage's task count relates to the configured one. 'matches': the configured count sized it
+// (or the count is unknown). 'coalesced': AQE merged the partitions into fewer tasks. 'adjusted': more
+// tasks than configured under AQE, which a skew-join split can produce and several shuffle reads in
+// one execution can obscure: the stage's own repartition(n) is blamed only when the plan holds one.
+// 'own': a different count with no AQE to rewrite it, or with a repartition(n) in the plan, which is
+// a repartition(n) or an RDD operation that fixed its own count.
+                                                                  
+
+function countRelation(stage               , basis                )                {
+  if (basis.count == null || stage.taskCount === basis.count) return 'matches';
+  if (basis.coalesce && stage.taskCount < basis.count) return 'coalesced';
+  if (basis.aqe && stage.taskCount > basis.count && (basis.skewSplit !== false || !basis.explicitRepartition)) return 'adjusted';
+  return 'own';
+}
+
+// Whether partition-count advice fits a shuffle-reading stage. The count that brings each shuffle
+// partition to the ideal size is per stage; the property is job-wide. 'sufficient': the tasks are
+// already at or under the ideal size. 'aqeCoalesced' and 'ownPartitioning' follow countRelation
+// ('ownPartitioning' also when the property already gives at least the count needed). Otherwise
+// the property limits the stage: 'raise'.
+function shufflePartitionCase(stage               , ctx             )                                                                                       {
+  const basis = partitionBasis(stage, ctx);
+  const { count, key } = basis;
   const needed = Math.ceil(stage.shuffleReadBytes / IDEAL_BYTES_PER_PARTITION_TASK);
   const tasks = Math.max(1, stage.taskCount);
+  const relation = countRelation(stage, basis);
   let partitions                    = 'raise';
   if (stage.shuffleReadBytes / tasks <= IDEAL_BYTES_PER_PARTITION_TASK) partitions = 'sufficient';
-  else if (count != null && stage.taskCount < count
-    && loggedAs(app, 'spark.sql.adaptive.enabled', true) && loggedAs(app, 'spark.sql.adaptive.coalescePartitions.enabled', true)) partitions = 'aqeCoalesced';
-  else if (count != null && count >= needed) partitions = 'ownPartitioning';
-  return { partitions, count, needed };
+  else if (relation === 'coalesced') partitions = 'aqeCoalesced';
+  else if (count != null && count >= needed && relation !== 'adjusted') partitions = 'ownPartitioning';
+  return { partitions, count, key, needed };
 }
 
 const ADVISORY_PARTITION_SIZE_KEY = 'spark.sql.adaptive.advisoryPartitionSizeInBytes';
+const PARALLELISM_FIRST_KEY = 'spark.sql.adaptive.coalescePartitions.parallelismFirst';
+
+// What sized a shuffle-reading stage's tasks, for advice that moves the partition count: the
+// countRelation outcomes as levers ('matches' is 'property'; 'adjusted' is 'aqeAdjusted', where the
+// advice stays on the property). Under 'aqeCoalesced' the property does not set the count: AQE
+// sizes the merged tasks from the advisory size and, with parallelismFirst (Spark's default), from
+// defaultParallelism too (ShufflePartitionsUtil.coalescePartitions and CoalesceShufflePartitions
+// at v3.5.0).
+                                                                                      
+
+function partitionLever(stage               , ctx             )                                                                                             {
+  const basis = partitionBasis(stage, ctx);
+  const relation = countRelation(stage, basis);
+  const lever                 = relation === 'matches' ? 'property'
+    : relation === 'coalesced' ? 'aqeCoalesced'
+    : relation === 'adjusted' ? 'aqeAdjusted' : 'ownPartitioning';
+  return { lever, count: basis.count, key: basis.key, explicitRepartition: basis.explicitRepartition };
+}
+
+// What follows advice on the property when the stage ran more tasks than configured under AQE.
+function aqeAdjustedNote(stage               , count               , explicitRepartition         )         {
+  return explicitRepartition
+    ? ` (this stage ran ${stage.taskCount} tasks against ${count} configured, and AQE skew-join splits add tasks, so that is not necessarily its own repartition(n))`
+    : ` (this stage ran ${stage.taskCount} tasks against ${count} configured, and the plan shows no repartition behind the difference)`;
+}
+
+// Whether parallelismFirst is on for the run: logged, else Spark's default from 3.2.0, where it
+// exists. A log that records no Spark version is read as current.
+function parallelismFirstOn(app                    )          {
+  const effective = effectiveConf(app, PARALLELISM_FIRST_KEY);
+  if (effective != null) return effective.toLowerCase() === 'true';
+  const version = /^(\d+)\.(\d+)/.exec(app?.sparkVersion ?? '');
+  return version == null || Number(version[1]) > 3 || Number(version[2]) >= 2;
+}
+
+// The conf that makes AQE's merged tasks larger. With parallelismFirst on, AQE targets
+// min(total / defaultParallelism, advisory size), so the advisory size alone merges nothing for
+// small tasks: turn parallelismFirst off first. With it off, the advisory size is the lever.
+function aqeCoalesceLargerFix(app                    )                                               {
+  return parallelismFirstOn(app)
+    ? { keys: `setting ${PARALLELISM_FIRST_KEY}=false (so AQE targets the advisory size, not defaultParallelism tasks) and then raising ${ADVISORY_PARTITION_SIZE_KEY}`, remediation: [setConf(PARALLELISM_FIRST_KEY, false), increaseConf(ADVISORY_PARTITION_SIZE_KEY)] }
+    : { keys: `raising ${ADVISORY_PARTITION_SIZE_KEY}`, remediation: [increaseConf(ADVISORY_PARTITION_SIZE_KEY)] };
+}
 
 // lowShuffleParallelism's fix. Unlike the shuffle finding it fires on stages whose tasks are larger
 // than the ideal size, so its cases are 'raise', 'aqeCoalesced' and 'ownPartitioning'.
-function lowShuffleParallelismFix(stage               , app                    )                                                                              {
-  const { partitions, count, needed } = shufflePartitionCase(stage, app);
+function lowShuffleParallelismFix(stage               , ctx             )                                                                              {
+  const { partitions, count, key, needed } = shufflePartitionCase(stage, ctx);
   if (partitions === 'aqeCoalesced') {
     return {
       partitions,
@@ -829,36 +1140,104 @@ function lowShuffleParallelismFix(stage               , app                    )
   if (partitions === 'ownPartitioning') {
     return {
       partitions,
-      text: `spark.sql.shuffle.partitions is already ${count}, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller`,
+      text: `${key} is already ${count}, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller`,
       remediation: [],
     };
   }
   return {
     partitions,
-    text: 'raise spark.sql.shuffle.partitions so each partition is smaller',
-    remediation: [increaseConf('spark.sql.shuffle.partitions', count == null ? null : needed)],
+    text: `raise ${key} so each partition is smaller`,
+    remediation: [increaseConf(key, count == null ? null : needed)],
   };
 }
 
-// A Spark byte-size property as bytes (a plain number, or with a k/m/g/t suffix, optionally 'b'), or
-// null when it is not one. Negative numbers pass through: -1 turns auto-broadcast off.
-function parseSparkBytes(value                    )                {
-  const m = /^(-?\d+(?:\.\d+)?)\s*([kmgt]?)b?$/i.exec(value?.trim() ?? '');
-  if (m == null) return null;
-  return m[2] === '' ? Number(m[1]) : Number(m[1]) * 1024 ** ('kmgt'.indexOf(m[2].toLowerCase()) + 1);
+// tinyTask's fix on a stage that reads a shuffle: fewer, larger tasks, through whatever sized this
+// stage. Lowering spark.sql.shuffle.partitions merges nothing AQE kept for parallelism, and does
+// not touch a repartition(n).
+function tinyShuffleFix(stage               , ctx             , coalesceTo        )                                               {
+  const { lever, count, key, explicitRepartition } = partitionLever(stage, ctx);
+  if (lever === 'aqeCoalesced') {
+    const larger = aqeCoalesceLargerFix(stageApp(ctx, stage));
+    return {
+      text: `${larger.keys}, or using .coalesce(${coalesceTo}): AQE already coalesced the ${count} configured shuffle partitions into ${stage.taskCount} tasks, so lowering the shuffle partition count will not merge them`,
+      remediation: larger.remediation,
+    };
+  }
+  if (lever === 'ownPartitioning') {
+    const hint = `lowering the repartition(n) or RDD partition count that sized this stage, or using .coalesce(${coalesceTo})`;
+    return { text: `${hint} (the configured shuffle partition count is ${count} but this stage ran ${stage.taskCount} tasks)`, remediation: [codeFix(hint)] };
+  }
+  const note = lever === 'aqeAdjusted' ? aqeAdjustedNote(stage, count, explicitRepartition) : '';
+  return { text: `lowering ${key} or using .coalesce(${coalesceTo})${note}`, remediation: [decreaseConf(key)] };
 }
 
-// The run's effective spark.sql.autoBroadcastJoinThreshold in bytes (logged, else Spark's 10 MiB);
-// negative when auto-broadcast is disabled, null when the logged value is not a size.
-function effectiveBroadcastThreshold(app                    )                {
-  const raw = effectiveConf(app, 'spark.sql.autoBroadcastJoinThreshold');
+// spill's fix on a stage that reads a shuffle and spills for volume: smaller partitions or more
+// memory. Raising spark.sql.shuffle.partitions only helps while the property sized the stage.
+function spillShuffleFix(stage               , ctx             )                                               {
+  const memory = increaseConf('spark.executor.memory');
+  const { lever, count, key, explicitRepartition } = partitionLever(stage, ctx);
+  if (lever === 'aqeCoalesced') {
+    return {
+      text: `AQE coalesced the ${count} configured shuffle partitions into ${stage.taskCount} tasks: lower ${ADVISORY_PARTITION_SIZE_KEY} so each partition is smaller, or increase executor memory`,
+      remediation: [decreaseConf(ADVISORY_PARTITION_SIZE_KEY), memory],
+    };
+  }
+  if (lever === 'ownPartitioning') {
+    const hint = "raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller";
+    return { text: `the configured shuffle partition count is ${count} but this stage ran ${stage.taskCount} tasks, so ${hint}, or increase executor memory`, remediation: [codeFix(hint), memory] };
+  }
+  const note = lever === 'aqeAdjusted' ? aqeAdjustedNote(stage, count, explicitRepartition) : '';
+  return { text: `raise ${key} or increase executor memory${note}`, remediation: [increaseConf(key), memory] };
+}
+
+// stageSlowness's fix on a stage that reads a shuffle: more tasks. spark.default.parallelism sizes
+// RDD shuffles only (the DataFrame shuffle reads spark.sql.shuffle.partitions), so it is not offered.
+function slowShuffleFix(stage               , ctx             )                                               {
+  const { lever, count, key, explicitRepartition } = partitionLever(stage, ctx);
+  if (lever === 'aqeCoalesced') {
+    return {
+      text: `AQE coalesced the ${count} configured shuffle partitions into ${stage.taskCount} tasks, so lower ${ADVISORY_PARTITION_SIZE_KEY} to get more of them`,
+      remediation: [decreaseConf(ADVISORY_PARTITION_SIZE_KEY)],
+    };
+  }
+  if (lever === 'ownPartitioning') {
+    const hint = "raise this stage's own partition count (its repartition(n) or RDD parallelism)";
+    return { text: `the configured shuffle partition count is ${count} but this stage ran ${stage.taskCount} tasks, so ${hint}`, remediation: [codeFix(hint)] };
+  }
+  const note = lever === 'aqeAdjusted' ? aqeAdjustedNote(stage, count, explicitRepartition) : '';
+  return { text: `raise ${key}${note}`, remediation: [increaseConf(key)] };
+}
+
+// Spark's default spark.dynamicAllocation.maxExecutors: no upper bound.
+const UNBOUNDED_EXECUTORS = 2147483647;
+
+const BROADCAST_THRESHOLD_KEY = 'spark.sql.autoBroadcastJoinThreshold';
+const ADAPTIVE_BROADCAST_THRESHOLD_KEY = 'spark.sql.adaptive.autoBroadcastJoinThreshold';
+// AQE keeps a join side out of a broadcast when too few of its shuffle partitions are non-empty
+// (DynamicJoinSelection at v3.5.0).
+const NON_EMPTY_PARTITION_RATIO_KEY = 'spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin';
+
+// The property that bounds a broadcast: with AQE, runtime sizes are judged against the adaptive
+// threshold when it is set (JoinSelectionHelper.canBroadcastBySize, apache/spark v3.5.0), and
+// against spark.sql.autoBroadcastJoinThreshold otherwise.
+function broadcastThresholdKey(app                    , aqe         )         {
+  return aqe && effectiveConf(app, ADAPTIVE_BROADCAST_THRESHOLD_KEY) !== undefined ? ADAPTIVE_BROADCAST_THRESHOLD_KEY : BROADCAST_THRESHOLD_KEY;
+}
+
+// The run's effective value of a broadcast threshold property in bytes (logged, else Spark's 10 MiB
+// default for the static one); negative when auto-broadcast is disabled, null when the value is
+// not a size or the property has none.
+function effectiveBroadcastThreshold(app                    , key        )                {
+  const raw = effectiveConf(app, key);
   if (raw != null && /^-\d+$/.test(raw)) return -1;
   return parseSparkBytes(raw);
 }
 
-// No dynamic-allocation property has an effect on a run whose logged conf turns it off.
+// No dynamic-allocation property has an effect on a run that has it off: Spark's default is off, so
+// an unset property counts as off, as it does for idleCapacityFix.
 function dynamicAllocationOff(app                    )          {
-  return app?.resources?.dynamicAllocationEnabled === false || app?.config?.['spark.dynamicAllocation.enabled']?.trim().toLowerCase() === 'false';
+  if (app?.resources?.dynamicAllocationEnabled != null) return !app.resources.dynamicAllocationEnabled;
+  return !loggedAs(app, 'spark.dynamicAllocation.enabled', true);
 }
 
 // Shared by cacheUtilization's two variants, worded per storage source: neither is a runtime
@@ -1110,8 +1489,7 @@ function gcConfidence(
 // warnFloor gates the finding, so a value just past it is the weakest evidence this detector can
 // produce. highFloor is critPct: straggler's own second (speculative-share) tier, 2x warnPct
 // (0.10 -> 0.20); reused as the high-confidence bar for the stragglerShare path too since that
-// metric has no dedicated critical tier of its own (see the "no dedicated critical tier" comment
-// on the straggler detector) but is the same 0-1 task-share magnitude.
+// metric has no critical tier of its own but is the same 0-1 task-share magnitude.
 function stragglerConfidence(shareValue        , warnFloor        , highFloor        )                            {
   if (shareValue < warnFloor * 1.5) return 'low';
   if (shareValue >= highFloor) return 'high';
@@ -1182,29 +1560,156 @@ function gcValidation(minRunTimeMs        )         {
 
 const INCOMPLETE_RUN_RECOMMENDATION = 'This event log never recorded an ApplicationEnd event: the capture stopped before the run finished (a job still running, a rotated log, or a cut-short capture), so every figure on this board covers only what was captured.';
 
+// straggler's own gate: tasks over 4x P50 are more than `shareWarn` of the stage (`shareWarnAtFloor`
+// when the occupancy-clipped tail clears the runtime floor), or a speculative task ran. Null when it
+// does not hold, else what the finding reports.
+function stragglerGate(stage               , ctx             , thresholds   
+                                                                                                  
+                                             
+ )                                                                                                                 {
+  if (stage.taskCount < thresholds.minTasks) return null;
+  const runMs = appDurationMs(ctx.app);
+  if (stageBelowRuntimeFloor(stage, ctx, thresholds.floorPctWarn)) return null;
+  const stragglerShare = (stage.stragglerCount ?? 0) / stage.taskCount;
+  const useSpeculative = (stage.speculativeTasks ?? 0) > 0;
+  if (!useSpeculative && stragglerShare <= thresholds.shareWarnAtFloor) return null;
+  const speculativeShare = useSpeculative ? stage.speculativeTasks / stage.taskCount : 0;
+  // The claim estimate() reports as savings: a high straggler/speculative share on a stage
+  // whose tasks barely vary models near-zero savings, so it must not outrank 'info'.
+  const floorWasteMs = tailClaimFloorMs(stragglerTailClaim(stage), stage.id, ctx);
+  const meetsWarnFloor = meetsRuntimeFloor(floorWasteMs, runMs, thresholds.floorPctWarn);
+  const meetsCritFloor = meetsRuntimeFloor(floorWasteMs, runMs, thresholds.floorPctCrit);
+  // The lower gate needs positive evidence the tail matters: meetsRuntimeFloor passes by
+  // default when the app's duration is unknown (an incomplete run), which isn't that.
+  const stragglerShareFires = stragglerShare > thresholds.shareWarn
+    || (stragglerShare > thresholds.shareWarnAtFloor && runMs != null && meetsWarnFloor);
+  if (!useSpeculative && !stragglerShareFires) return null;
+  const speculativeTier = speculativeShare >= thresholds.critPct && meetsCritFloor ? 'critical'
+                         : speculativeShare >= thresholds.warnPct && meetsWarnFloor ? 'warning' : 'info';
+  // A straggler share has only a warning tier (threshold-tables.md, Straggler row).
+  const stragglerTier = stragglerShareFires && meetsWarnFloor ? 'warning' : 'info';
+  // Report whichever signal actually drove the finding, not just whether speculation was on:
+  // a high stragglerShare with few speculative retries must not be reported as a low-value
+  // speculativeTasks count. Ties keep the speculative-driven default.
+  const useSpeculativeMetric = useSpeculative && !(IMPACT_BAND_ORDER[stragglerTier] < IMPACT_BAND_ORDER[speculativeTier]);
+  const value = useSpeculativeMetric ? stage.speculativeTasks : Math.round(stragglerShare * 100);
+  return {
+    useSpeculativeMetric, value,
+    detail: useSpeculativeMetric
+      ? `${value} speculative attempt${value === 1 ? '' : 's'} discarded${speculationRuleNote(ctx.app)}`
+      : `${value}% of tasks ran over ${STRAGGLER_FACTOR}× the median`,
+    confidence: useSpeculativeMetric
+      ? stragglerConfidence(speculativeShare, thresholds.warnPct, thresholds.critPct)
+      : stragglerConfidence(stragglerShare, thresholds.shareWarn, thresholds.critPct),
+  };
+}
+
+// The straggler finding's evidence, recommendation and fix for the tail's cause. Without a measured
+// cause there is nothing to fix yet, only the causes to check.
+function stragglerAdvice(stage               , ctx             , tail             , detail        )   
+                                                                                                                             
+                         
+                                         
+  {
+  const a = tail.attribution;
+  const cpuPct = a?.cpuPct != null ? Math.round(a.cpuPct) : undefined;
+  const evidence = { cause: tail.cause, ...(tail.cause === 'unattributed' ? {} : { causeSharePct: tail.sharePct }), ...(cpuPct !== undefined ? { cpuPct } : {}) };
+  const share = `${tail.sharePct}% of the slow tasks' extra time`;
+  switch (tail.cause) {
+    case 'data': {
+      // skew reports a tail its gate admits; this is one only the straggler gate does.
+      const fix = skewFix(stage, ctx, false, tail);
+      const ratio = a?.dataRatio != null ? Math.round(a.dataRatio * 10) / 10 : null;
+      return {
+        evidence: { ...evidence, origin: fix.origin, aqeSkew: fix.aqeSkew }, remediation: fix.remediation,
+        recommendation: `${detail}: data volume accounts for ${share} (${ratio !== null ? `the slow tasks read a median ${ratio}× the data of the median task` : 'the median task read almost nothing'}): ${fix.text}.`,
+      };
+    }
+    case 'gc':
+      return {
+        evidence, remediation: [increaseConf('spark.executor.memory')],
+        recommendation: `${detail}: GC accounts for ${share}: reduce object creation, use primitive types, avoid UDFs, increase executor memory.`,
+      };
+    case 'fetchWait':
+      return {
+        evidence, remediation: undefined,
+        recommendation: `${detail}: waiting on shuffle fetches accounts for ${share}: look for a slow or overloaded node serving shuffle blocks, executors lost mid-stage, or reducers fetching many small blocks.`,
+      };
+    case 'host': {
+      const speculation = switchFix(loggedAs(ctx.app, 'spark.speculation', true), 'spark.speculation', true,
+        'check what it was running, and consider enabling spark.speculation to relaunch a lagging task automatically',
+        'check what it was running; speculation is already on, so a lagging task there is already relaunched');
+      return {
+        evidence: { ...evidence, host: a .host , hostTasks: a .hostTasks }, remediation: speculation.remediation,
+        recommendation: `${detail}: ${a .hostTasks} of the ${a .tasks} slow tasks ran on ${a .host}, which accounts for ${share}: ${speculation.text}.`,
+      };
+    }
+    case 'unexplained': {
+      const cpu = cpuPct === undefined ? '' : cpuPct < 50
+        ? ` They used ${cpuPct}% of their run time on CPU, so they mostly waited (on storage, a remote service or a lock).`
+        : ` They used ${cpuPct}% of their run time on CPU, so the work itself is slow.`;
+      // A tail that read clearly more data is not "no more data" even when run time did not scale
+      // with it; where data is the largest share anyway, the data advice stays.
+      const dataRatio = a?.dataRatio ?? 0;
+      if (a != null && dataRatio >= MORE_DATA_RATIO) {
+        const rounded = Math.round(dataRatio * 10) / 10;
+        const dataShare = Math.round((a.dataMs / a.excessMs) * 100);
+        const dataLargest = a.dataMs >= a.excessMs - a.dataMs - a.gcMs - a.fetchWaitMs - a.hostMs;
+        const lead = `${detail}: the slow tasks read a median ${rounded}× the data of the median task, and data volume accounts for ${dataShare}% of their extra time`;
+        if (dataLargest) {
+          const fix = skewFix(stage, ctx, false, tail);
+          return {
+            evidence: { ...evidence, origin: fix.origin, aqeSkew: fix.aqeSkew }, remediation: fix.remediation,
+            recommendation: `${lead}, more than anything else the log attributes (not a majority): ${fix.text}.`,
+          };
+        }
+        return {
+          evidence, remediation: undefined,
+          recommendation: `${lead}, so run time did not scale with it, and GC, shuffle fetch wait and one slow host do not account for the rest.${cpu} Look at per-record cost (UDFs, regular expressions, a call out per row) as well as the data they read.`,
+        };
+      }
+      return {
+        evidence, remediation: undefined,
+        recommendation: `${detail}: the slow tasks read no more data than the median task, and GC, shuffle fetch wait and one slow host do not account for their time.${cpu} Look at per-record cost (UDFs, regular expressions, a call out per row).`,
+      };
+    }
+    default:
+      return {
+        evidence, remediation: undefined,
+        recommendation: `${detail}: nothing in the log attributes the slow tasks to data volume, GC, shuffle fetch wait or one host, so check their input sizes, GC time and hosts before choosing a fix.`,
+      };
+  }
+}
+
 export const DETECTORS = [
   defineStageDetector({
     type: 'skew', order: 30, fixEffort: 'code', version: 1,
     emits: ['skew'],
     docAnchor: '#bottleneck-skew',
-    thresholds: { ratioWarn: 3, minTasksForP95: 20, floorPctWarn: IMPACT_FLOOR_PCT_WARN },
+    // dataShareMin: the share of the slow tasks' extra time that run time proportional to their data
+    // volume must account for before the tail is data skew; straggler judges a tail with it too.
+    thresholds: { ratioWarn: 3, minTasksForP95: 20, dataShareMin: 0.5, floorPctWarn: IMPACT_FLOOR_PCT_WARN },
     detect(stage, ctx, thresholds)                 {
-      const result = computeSkewRatio(stage, thresholds.minTasksForP95);
-      if (result === null) return null;
-      const { ratio, metric } = result;
-      if (ratio <= thresholds.ratioWarn) return null;
-      // The claim estimate() reports as savings, clipped the same way, so the gate agrees with it.
-      const floorWasteMs = tailClaimFloorMs(skewTailClaim(stage, metric === 'P95/median'), stage.id, ctx);
-      if (!meetsRuntimeFloor(floorWasteMs, appDurationMs(ctx.app), thresholds.floorPctWarn)) return null;
+      const gate = skewGate(stage, ctx, thresholds.ratioWarn, thresholds.minTasksForP95, thresholds.floorPctWarn);
+      if (gate === null) return null;
+      const { ratio, metric } = gate;
+      // A tail data does not explain, or one with no measured cause, is not skew: straggler reports
+      // it, with the cause (it re-runs this same gate, see skewGate).
+      const tail = tailVerdict(stage, thresholds.dataShareMin);
+      if (tail.cause !== 'data') return null;
       const value = Math.round(ratio * 10) / 10;
-      const fix = skewFix(stage, ctx);
+      const fix = skewFix(stage, ctx, false, tail);
+      const dataRatio = tail.attribution?.dataRatio != null
+        ? Math.round(tail.attribution.dataRatio * 10) / 10 : undefined;
       return {
-        type: 'skew', stageId: stage.id, origin: fix.origin,
+        type: 'skew', stageId: stage.id, origin: fix.origin, aqeSkew: fix.aqeSkew,
+        cause: tail.cause,
+        ...(dataRatio !== undefined ? { dataRatio } : {}),
         impactBand: 'warning',
         metric, value,
         confidence: skewConfidence(ratio, thresholds.ratioWarn),
         validationRequired: `Flagged only when it costs at least ${shareLabel(thresholds.floorPctWarn)} of run time.`,
-        recommendation: `Task duration ratio (${metric}) is ${value}×: ${fix.text}.`,
+        recommendation: `Task duration ratio (${metric}) is ${value}×${dataRatio !== undefined ? `, and the slow tasks read a median ${dataRatio}× the data of the median task` : ', and the median task read almost nothing'}: ${fix.text}.`,
         remediation: fix.remediation,
       };
     },
@@ -1221,8 +1726,7 @@ export const DETECTORS = [
     emits: ['stageShape'],
     docAnchor: '#bottleneck-stage-shape',
     // lowParallelismFloorPct: the same 0.5% runtime floor the tiered detectors use. Parallelizing a
-    // stage can't save more than the stage's own duration, so a shorter stage can't clear it; on
-    // the 14 real logs that was 2839 of 3005 lowParallelism findings (2168 on sub-second stages).
+    // stage can't save more than the stage's own duration, so a shorter stage can't clear it.
     // App-wide idle capacity stays covered by utilization.
     // taskStageSkew: stageShareMin is the share of the stage's wall-clock the longest task must
     // span, and skewWarn how far past the median task it must run (skew's own max/median 3×).
@@ -1304,10 +1808,13 @@ export const DETECTORS = [
         // Cores idle during the straggler's tail, at achieved concurrency (not full cluster
         // capacity, which is lowParallelism's territory): resourceOnly, since skew and straggler
         // already claim that tail's wall-clock time.
-        const idleCoreMs =
+        const tailIdleCoreMs =
           Math.max(0, Math.min(totalCores, taskCount) - 1) *
           Math.max(0, (stage.taskDurationMax ?? 0) - (stage.taskDurationP50 ?? 0));
-        return costOnly('measured', { value: idleCoreMs, unit: 'coreMs', idle: true });
+        // An upper bound that assumes every executor stays allocated through the tail, so it is
+        // modeled; one stage cannot idle more than the whole run did, so it is clipped to that.
+        const idleCoreMs = ctx.runIdleCoreMs == null ? tailIdleCoreMs : Math.min(tailIdleCoreMs, ctx.runIdleCoreMs);
+        return costOnly('modeled', { value: idleCoreMs, unit: 'coreMs', idle: true });
       }
       return null;
     },
@@ -1317,22 +1824,21 @@ export const DETECTORS = [
     emits: ['shuffle'],
     docAnchor: '#bottleneck-shuffle',
     // stageFloorPct: the 0.5% runtime floor. The shuffle claim is clipped to the stage, so on a
-    // shorter stage it graded info: 182 of 284 shuffle findings on the 14 real logs. The shuffle
-    // is still there on those stages; the floor is why they're dropped.
+    // shorter stage it would grade info; the shuffle is still there, the floor drops the finding.
     thresholds: { minBytes: 50 * MB, stageFloorPct: 0.005 },
     detect(stage, ctx, thresholds)                 {
       const bytes = stage.shuffleReadBytes;
       if (bytes <= thresholds.minBytes) return null;
       if (stageBelowRuntimeFloor(stage, ctx, thresholds.stageFloorPct)) return null;
-      const { partitions, count } = shufflePartitionCase(stage, ctx.app);
+      const { partitions, count, key } = shufflePartitionCase(stage, ctx);
       const perTask = formatBytes(bytes / Math.max(1, stage.taskCount));
       const fix = partitions === 'sufficient'
         ? { text: `its ${stage.taskCount} tasks already read about ${perTask} each, so more partitions will not help: consider a broadcast join for the smaller side`, remediation: [] }
         : partitions === 'aqeCoalesced'
           ? { text: `AQE coalesced it into ${stage.taskCount} tasks (from ${count} configured partitions): consider lowering ${ADVISORY_PARTITION_SIZE_KEY} or adding a broadcast join`, remediation: [decreaseConf(ADVISORY_PARTITION_SIZE_KEY)] }
           : partitions === 'ownPartitioning'
-            ? { text: `spark.sql.shuffle.partitions is already ${count}, so consider raising this stage's own partition count (its repartition(n) or RDD parallelism) or adding a broadcast join`, remediation: [] }
-            : { text: 'consider increasing spark.sql.shuffle.partitions or adding a broadcast join', remediation: [increaseConf('spark.sql.shuffle.partitions')] };
+            ? { text: `${key} is already ${count}, so consider raising this stage's own partition count (its repartition(n) or RDD parallelism) or adding a broadcast join`, remediation: [] }
+            : { text: `consider increasing ${key} or adding a broadcast join`, remediation: [increaseConf(key)] };
       return {
         type: 'shuffle', stageId: stage.id,
         impactBand: 'info', partitions,
@@ -1352,8 +1858,11 @@ export const DETECTORS = [
       const measuredMs = fetchWaitWallClockMs(stage);
       const wasteMs = measuredMs == null ? modeledMs : Math.min(modeledMs, measuredMs);
       // rawWaste: the measured byte volume behind the modeled figure.
-      return singleStageImpact(wasteMs, finding.stageId, ctx,
-        measuredMs != null && measuredMs < modeledMs ? 'measured' : 'modeled', { value: shuffleReadBytes, unit: 'bytes' });
+      const capped = measuredMs != null && measuredMs < modeledMs;
+      return withAssumptions(
+        singleStageImpact(wasteMs, finding.stageId, ctx, capped ? 'measured' : 'modeled', { value: shuffleReadBytes, unit: 'bytes' }),
+        `${ioModelAssumption(SHUFFLE_THROUGHPUT_BPS, stage)}; counts network transfer only${capped ? ', capped at the fetch wait the tasks measured' : ''}`,
+      );
     },
   }),
   defineStageDetector({
@@ -1373,13 +1882,13 @@ export const DETECTORS = [
         const fix = skewFix(stage, ctx, true);
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
-          rule: 'shufflePartitionSkew', origin: fix.origin, metric: 'shuffleReadMax', value: max,
+          rule: 'shufflePartitionSkew', origin: fix.origin, aqeSkew: fix.aqeSkew, metric: 'shuffleReadMax', value: max,
           recommendation: `The largest shuffle partition (${formatBytes(max)}) is ${ratioText}: ${fix.text}.`,
           remediation: fix.remediation,
         });
       }
       if (total >= thresholds.lowParTotalBytes && taskCount <= thresholds.lowParMaxTasks) {
-        const fix = lowShuffleParallelismFix(stage, ctx.app);
+        const fix = lowShuffleParallelismFix(stage, ctx);
         out.push({
           type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
           rule: 'lowShuffleParallelism', partitions: fix.partitions, metric: 'taskCount', value: taskCount,
@@ -1442,8 +1951,7 @@ export const DETECTORS = [
     type: 'spill', order: 10, fixEffort: 'code', version: 1,
     emits: ['spill'],
     docAnchor: '#bottleneck-spill',
-    // stageFloorPct: the 0.5% runtime floor, as for shuffle (12 of 38 spill findings on the 14 real
-    // logs, all info). The spill is still there on those stages; the floor is why they're dropped.
+    // stageFloorPct: the 0.5% runtime floor, as for shuffle.
     thresholds: { singleTaskDiskGiB: 1, singleTaskMemGiB: 4, highDiskGiB: 1, highTaskDiskMB: 512, highMemGiB: 4, medDiskMB: 256, medMemGiB: 1, skewRatio: 5, skewDiskFloorMB: 128, skewMemFloorMB: 256, skewMinTasks: 10, stageFloorPct: 0.005 },
     detect(stage, ctx, thresholds)                 {
       if (stage.memoryBytesSpilled === 0) return null;
@@ -1454,7 +1962,7 @@ export const DETECTORS = [
       const impactBand = 'warning';
       // shuffle.partitions sizes a shuffle's reduce side: a stage that reads no shuffle gets memory only.
       const reads = stageReads(stage);
-      const partitionAdvice = reads === 'shuffle';
+      const shuffleFix = reads === 'shuffle' ? spillShuffleFix(stage, ctx) : null;
       return {
         type: 'spill', stageId: stage.id, impactBand, reads,
         spillMagnitude: mag?.magnitude,
@@ -1465,10 +1973,10 @@ export const DETECTORS = [
           : 'Spill cause could not be classified: inspect per-task spill metrics in the Spark UI before acting.',
         recommendation: cls === 'skew'
           ? `${formatBytes(stage.memoryBytesSpilled)} spilled, skew-driven: fix task skew first; adding memory will not help.`
-          : partitionAdvice
-            ? `${formatBytes(stage.memoryBytesSpilled)} spilled: raise spark.sql.shuffle.partitions or increase executor memory.`
+          : shuffleFix
+            ? `${formatBytes(stage.memoryBytesSpilled)} spilled: ${shuffleFix.text}.`
             : `${formatBytes(stage.memoryBytesSpilled)} spilled: this stage reads no shuffle, so increase executor memory or process less data per task.`,
-        remediation: cls === 'skew' ? [] : [...(partitionAdvice ? [increaseConf('spark.sql.shuffle.partitions')] : []), increaseConf('spark.executor.memory')],
+        remediation: cls === 'skew' ? [] : shuffleFix?.remediation ?? [increaseConf('spark.executor.memory')],
       };
     },
     estimate(finding, ctx)                        {
@@ -1478,7 +1986,10 @@ export const DETECTORS = [
       const diskBytesSpilled = stage.diskBytesSpilled ?? 0;
       const wasteMs = (diskBytesSpilled / (SPILL_IO_THROUGHPUT_BPS * stageIoParallelism(stage))) * 1000;
       // Surfaces the number the formula uses: the displayed metric is memoryBytesSpilled, but disk spill costs the I/O time.
-      return singleStageImpact(wasteMs, finding.stageId, ctx, 'modeled', { value: diskBytesSpilled, unit: 'bytes' });
+      return withAssumptions(
+        singleStageImpact(wasteMs, finding.stageId, ctx, 'modeled', { value: diskBytesSpilled, unit: 'bytes' }),
+        `${ioModelAssumption(SPILL_IO_THROUGHPUT_BPS, stage)}; counts disk spill only, not memory spill, serialization or GC`,
+      );
     },
   }),
   defineStageDetector({
@@ -1487,14 +1998,13 @@ export const DETECTORS = [
     docAnchor: '#bottleneck-gc',
     thresholds: {
       warnPct100: 10,
-      // Descending tier: ExecutorGcHeuristic, ported as-is.
+      // A GC share below this reads as over-provisioned memory (the low-GC note).
       lowInfoPct100: 5,
-      // NOT SOURCED: our own noise floor so a stage that barely ran doesn't flag either direction.
+      // Noise floor so a stage that barely ran doesn't flag either direction.
       minRunTimeMs: 10000,
       // lowInfoFloorPct: the 0.5% runtime floor stageShape's lowParallelism uses. The low-GC note is
       // an app-level memory-sizing signal; on a stage shorter than this share of the run it adds
-      // nothing to that call. On the 14 real logs that was 464 of 685 low-GC notes (of 701 gc
-      // findings); the low-GC pattern is still true on those stages, the floor is why they're dropped.
+      // nothing to that call.
       lowInfoFloorPct: 0.005,
     },
     detect(stage, ctx, thresholds)                 {
@@ -1511,8 +2021,14 @@ export const DETECTORS = [
           remediation: [increaseConf('spark.executor.memory')],
         };
       }
-      // Low-GC (cost) branch: only for stages that ran long enough to be meaningful.
+      // Low-GC (cost) branch: only for stages that ran long enough to be meaningful, and only as
+      // the fallback sizing signal: a log with measured executor heap peaks and a known executor
+      // memory gets memoryUtilization's heapOverProvisioned judgement instead (low GC time just
+      // means a low allocation rate).
       if ((stage.executorRunTime ?? 0) >= thresholds.minRunTimeMs
+          && !heapJudgementAvailable(ctx)
+          // A stage that failed, or lost tasks, ran too little work to call its GC low.
+          && stage.stageFailureReason == null && stage.failedTasks === 0
           && pct < thresholds.lowInfoPct100
           && !stageBelowRuntimeFloor(stage, ctx, thresholds.lowInfoFloorPct)) {
         const value = Math.round(pct * 10) / 10;
@@ -1551,7 +2067,7 @@ export const DETECTORS = [
     },
   }),
   defineStageDetector({
-    type: 'slowHost', order: 60, fixEffort: 'config', version: 1,
+    type: 'slowHost', order: 60, fixEffort: 'config', version: 2,
     emits: ['slowHost'],
     docAnchor: '#bottleneck-slow-host',
     thresholds: {
@@ -1562,11 +2078,15 @@ export const DETECTORS = [
       floorMs: 1000, floorBytes: 64 * MB,
       // stageFloorPct: the tiered detectors' 0.5% runtime floor. On a stage shorter than this share
       // of the run a slow host can't cost that much: duration findings are clipped to the stage
-      // and graded info, and a byte-dimension imbalance (no time estimate, so its ratio tier was
-      // its band) was graded info here since 6298149. So the stage is skipped: on the 14 real logs
-      // 324 of 452 slowHost findings, all info. The imbalance is still there on those stages; the
-      // floor is why they're dropped.
+      // and would grade info, and a byte-dimension imbalance (no time estimate, so its ratio tier
+      // is its band) is graded info there too. So the stage is skipped.
       stageFloorPct: 0.005,
+      // outlierAlpha: the nominal chance a stage of alike tasks gets any executor flagged on task
+      // time. The slowest of many executors always tops the median, so a task-time executor is
+      // flagged only when its mean sits further above the median than the stage's own task spread
+      // and its task count allow (chanceOutlierZ), more strictly the more executors there are.
+      // The normal-tail bound behind that cutoff is loose: simulated alike stages flag well under it.
+      outlierAlpha: 0.005,
     },
     detect(stage, ctx, thresholds)                   {
       const hosts = stage.hostStats ?? [];
@@ -1574,9 +2094,10 @@ export const DETECTORS = [
       if ((hosts.length < thresholds.minHosts && execs0.length < thresholds.minHosts) || stage.taskCount < thresholds.minTasks) return null;
       if (stageBelowRuntimeFloor(stage, ctx, thresholds.stageFloorPct)) return null;
       const out            = [];
+      const rule = speculationRule(ctx.app);
       const speculation = switchFix(loggedAs(ctx.app, 'spark.speculation', true), 'spark.speculation', true,
-        'check what it was running, and consider enabling spark.speculation to relaunch a lagging task automatically',
-        'check what it was running; speculation is already on, so a lagging task there is already relaunched');
+        `check what it was running, and consider enabling spark.speculation to relaunch a lagging task automatically${rule ? ` (with this run's settings, ${rule})` : ''}`,
+        `check what it was running; speculation is already on, so a lagging task there is already relaunched${rule ? ` (with this run's settings, ${rule})` : ''}`);
       if (hosts.length >= thresholds.minHosts) {
         const means = hosts.map(h => ({ host: h.host, taskCount: h.taskCount, mean: h.totalDuration / h.taskCount }));
         const sorted = [...means].map(h => h.mean).sort((a, b) => a - b);
@@ -1622,8 +2143,15 @@ export const DETECTORS = [
       const impactBandFor = (r        )                                         =>
         r >= tiers[3] ? 'critical' : (r >= tiers[1] ? 'warning' : (r >= tiers[0] ? 'info' : null));
       const floorMs = thresholds.floorMs, floorBytes = thresholds.floorBytes;
-      const dims                                                                                                                       = [
-        { dimension: 'taskTime', floor: floorMs, samples: execs.filter(e => e.taskCount > 0).map(e => ({ key: e.executorId, value: e.totalDuration / e.taskCount })) },
+      const taskTimeSamples = execs.filter(e => e.taskCount > 0).map(e => ({ key: e.executorId, value: e.totalDuration / e.taskCount }));
+      const taskTimeZs = executorDurationZ(execs);
+      const taskTimeZ = chanceOutlierZ(taskTimeSamples.length, thresholds.outlierAlpha);
+      const dims                                                                                                                                                                                        = [
+        {
+          dimension: 'taskTime', floor: floorMs, samples: taskTimeSamples,
+          // Only an executor the stage's own spread can't explain: with no known spread, none.
+          qualifies: sample => (taskTimeZs?.get(sample.key) ?? -Infinity) >= taskTimeZ,
+        },
         { dimension: 'inputBytes', floor: floorBytes, samples: execs.map(e => ({ key: e.executorId, value: e.inputBytes ?? 0 })) },
         { dimension: 'shuffleBytes', floor: floorBytes, samples: execs.map(e => ({ key: e.executorId, value: (e.shuffleReadBytes ?? 0) + (e.shuffleWriteBytes ?? 0) })) },
       ];
@@ -1633,14 +2161,13 @@ export const DETECTORS = [
         dims.push({ dimension: 'storageMemory', floor: floorBytes, samples: [...em.entries()].map(([id, m]) => ({ key: id, value: (m.onHeapStorageMemory ?? 0) + (m.offHeapStorageMemory ?? 0) })) });
       }
       for (const d of dims) {
-        const r = maxMedianRatio(d.samples);
+        const r = maxMedianRatio(d.samples, d.qualifies);
         if (!r || r.value < d.floor) continue; // absolute-magnitude floor: same ratio+floor shape as computeSpillMagnitude
         const tier = impactBandFor(r.ratio);
         if (!tier) continue;
-        // taskTime is the only wallClock-bearing dimension here: fixed fallback
-        // (its current floor case), overwritten by deriveImpactBand whenever this
-        // finding gets a real wallClock estimate. The other three dimensions never
-        // get a wallClock estimate, so they keep the dynamic tier unchanged.
+        // taskTime is the only wallClock-bearing dimension: it takes the fixed fallback band (see
+        // threshold-tables.md). The other three never get a wallClock estimate, so they keep the
+        // ratio tier.
         const impactBand = d.dimension === 'taskTime' ? 'info' : tier;
         out.push({
           type: 'slowHost', stageId: stage.id, impactBand,
@@ -1650,6 +2177,7 @@ export const DETECTORS = [
           // in this dimension's own unit (ms for taskTime, bytes for the rest).
           execMaxValue: r.value,
           executorId: r.key,
+          ...(d.dimension === 'taskTime' ? executorTaskTimeCaveat(r.key, r.ratio, d.samples.length, stage.taskCount) : {}),
           recommendation: `Executor ${r.key}'s ${SLOW_HOST_DIMENSION_LABEL[d.dimension]} is ${Math.round(r.ratio * 10) / 10}× the median: investigate uneven partition assignment or a degraded executor.`,
         });
       }
@@ -1683,7 +2211,7 @@ export const DETECTORS = [
     thresholds: { infoMin: 15 },
     // A stage slowHost already explains needs no generic "this stage is slow" finding on top.
     suppressedBy: 'slowHost',
-    detect(stage, _ctx, thresholds)                 {
+    detect(stage, ctx, thresholds)                 {
       // Basis is real wall-clock stage duration, not per-executor average; the impact-estimator
       // formula reuses this exact stageDurationMs computation.
       const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
@@ -1693,11 +2221,12 @@ export const DETECTORS = [
       const impactBand = durationMinutes >= t.infoMin ? 'info' : null;
       if (!impactBand) return null;
       const value = Math.round(durationMinutes * 10) / 10;
-      // shuffle.partitions and default.parallelism size a shuffle's reduce side: a stage that reads
-      // no shuffle gets the input-partitioning remedy instead.
+      // A shuffle's reduce-side partition count is the remedy on a stage that reads a shuffle; a stage
+      // that reads input files gets the input-partitioning remedy instead.
       const reads = stageReads(stage);
-      const recommendation = reads === 'shuffle'
-        ? `This stage ran ${value} minutes with no more specific cause flagged: often a partition-count problem, raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.`
+      const shuffleFix = reads === 'shuffle' ? slowShuffleFix(stage, ctx) : null;
+      const recommendation = shuffleFix
+        ? `This stage ran ${value} minutes with no more specific cause flagged: often a partition-count problem, ${shuffleFix.text}, or check for a large per-task data volume driving heavy shuffle and spill.`
         : reads === 'input'
           ? `This stage ran ${value} minutes with no more specific cause flagged and reads mostly input files: often too few or too uneven input partitions, so check input file sizes and lower spark.sql.files.maxPartitionBytes, or look for a large per-task data volume driving heavy spill.`
           : `This stage ran ${value} minutes with no more specific cause flagged and reads neither shuffle nor input files: check what it computes and for a large per-task data volume driving heavy spill.`;
@@ -1705,8 +2234,8 @@ export const DETECTORS = [
         type: 'stageSlowness', stageId: stage.id, impactBand,
         metric: 'stageDurationMinutes', value, reads,
         recommendation,
-        remediation: reads === 'shuffle'
-          ? [increaseConf('spark.sql.shuffle.partitions'), increaseConf('spark.default.parallelism')]
+        remediation: shuffleFix
+          ? shuffleFix.remediation
           : reads === 'input' ? [decreaseConf('spark.sql.files.maxPartitionBytes')] : [],
       };
     },
@@ -1721,8 +2250,7 @@ export const DETECTORS = [
       // too, hence TAIL_CLAIM's post-fix floor. Unknown cluster size: no defensible figure.
       if (ctx.totalCores <= 0) return costOnly('modeled');
       // A stage that read no input and no shuffle, its tasks idle waiting on an external system,
-      // gains nothing from more partitions (a 1-task JDBC count stage open 27 minutes on 5s of CPU
-      // was claimed 99% recoverable): claim 0. Stages that read bytes keep their claim.
+      // gains nothing from more partitions: claim 0. Stages that read bytes keep their claim.
       const readBytes = (stage.inputBytes ?? 0) + (stage.shuffleReadBytes ?? 0);
       const activeMs = typeof stage.taskActiveMs === 'number'
         ? stage.taskActiveMs
@@ -1793,63 +2321,39 @@ export const DETECTORS = [
     // floorPctWarn/floorPctCrit default to impact-band.ts's run-wide noise floor, so a tail this
     // gate admits at its warn floor grades at least warning there too.
     // shareWarnAtFloor: in a large stage, the few stragglers that gate it for tens of seconds can be
-    // only 2.5-5% of its tasks. Scored against a task-level replay of every stage on 14 real logs
-    // (recoverable = replay with each task over 4x P50 capped at P50), admitting 2.5-5% shares
-    // whose clipped tail already clears floorPctWarn found 3 such stages (10-48s) for 1 borderline
-    // miss; admitting every 2.5% share instead added 86 findings below the floor.
+    // only 2.5-5% of its tasks, so a 2.5% share counts once its clipped tail clears floorPctWarn.
     // A stage shorter than floorPctWarn of the run is skipped outright: its tail can't cost more
-    // than the stage's own duration, so every finding there graded info. On the 14 real logs that
-    // was 671 of 753 straggler findings, none above info; the slow tail is still real on those
-    // stages, the floor is why they're dropped.
+    // than the stage's own duration, so every finding there would grade info.
+    // A tail skew's gate admits and data explains is skew's finding; one it admits whose cause is not
+    // data is this one's. Both use skew's thresholds (ctx.skewThresholds), so tuning skew moves both.
     thresholds: { minTasks: 10, shareWarn: 0.05, shareWarnAtFloor: 0.025, warnPct: 0.10, critPct: 0.20, floorPctWarn: IMPACT_FLOOR_PCT_WARN, floorPctCrit: IMPACT_FLOOR_PCT_CRIT },
     detect(stage, ctx, thresholds)                 {
-      if (stage.taskCount < thresholds.minTasks) return null;
-      const runMs = appDurationMs(ctx.app);
-      if (stageBelowRuntimeFloor(stage, ctx, thresholds.floorPctWarn)) return null;
-      const stragglerShare = (stage.stragglerCount ?? 0) / stage.taskCount;
-      const useSpeculative = (stage.speculativeTasks ?? 0) > 0;
-      if (!useSpeculative && stragglerShare <= thresholds.shareWarnAtFloor) return null;
-      const speculativeShare = useSpeculative ? stage.speculativeTasks / stage.taskCount : 0;
-      // The claim estimate() reports as savings: a high straggler/speculative share on a stage
-      // whose tasks barely vary models near-zero savings, so it must not outrank 'info'.
-      const floorWasteMs = tailClaimFloorMs(stragglerTailClaim(stage), stage.id, ctx);
-      const meetsWarnFloor = meetsRuntimeFloor(floorWasteMs, runMs, thresholds.floorPctWarn);
-      const meetsCritFloor = meetsRuntimeFloor(floorWasteMs, runMs, thresholds.floorPctCrit);
-      // The lower gate needs positive evidence the tail matters: meetsRuntimeFloor passes by
-      // default when the app's duration is unknown (an incomplete run), which isn't that.
-      const stragglerShareFires = stragglerShare > thresholds.shareWarn
-        || (stragglerShare > thresholds.shareWarnAtFloor && runMs != null && meetsWarnFloor);
-      if (!useSpeculative && !stragglerShareFires) return null;
-      const speculativeTier = speculativeShare >= thresholds.critPct && meetsCritFloor ? 'critical'
-                             : speculativeShare >= thresholds.warnPct && meetsWarnFloor ? 'warning' : 'info';
-      // Straggler share has no dedicated critical tier per detector-contract.md; only warning.
-      const stragglerTier = stragglerShareFires && meetsWarnFloor ? 'warning' : 'info';
-      // Fixed fallback: overwritten by deriveImpactBand when this finding gets a real wallClock
-      // estimate (the common case). Only surfaces on the rare occupancy-sweep miss.
-      const impactBand = 'info';
-      // Report whichever signal actually drove the finding, not just whether speculation was on:
-      // a high stragglerShare with few speculative retries must not be reported as a low-value
-      // speculativeTasks count. Ties keep the speculative-driven default.
-      const useSpeculativeMetric = useSpeculative && !(IMPACT_BAND_ORDER[stragglerTier] < IMPACT_BAND_ORDER[speculativeTier]);
-      const value = useSpeculativeMetric ? stage.speculativeTasks : Math.round(stragglerShare * 100);
-      const detail = useSpeculativeMetric
-        ? `${value} speculative attempt${value === 1 ? '' : 's'} discarded`
-        : `${value}% of tasks straggled`;
-      // The skew advice fits only a stage that reads a shuffle feeding a join (skewFix).
-      const fix = skewFix(stage, ctx);
+      const skew = ctx.skewThresholds;
+      const tail = tailVerdict(stage, skew.dataShareMin);
+      const skewTail = skewGate(stage, ctx, skew.ratioWarn, skew.minTasksForP95, skew.floorPctWarn);
+      if (tail.cause === 'data' && skewTail !== null) return null; // skew reports it
+      // A tail skew's duration gate admits but data does not explain (or nothing attributes) is this finding's.
+      const fromSkewGate = skewTail !== null && tail.cause !== 'data';
+      const own = stragglerGate(stage, ctx, thresholds);
+      if (own === null && !fromSkewGate) return null;
+      // Through skew's gate alone the tail can be a share too small to state, so skew's ratio describes it.
+      const reported = own !== null
+        ? { metric: own.useSpeculativeMetric ? 'speculativeTasks' : 'stragglerShare', value: own.value, unit: own.useSpeculativeMetric ? 'count'          : 'pct'         , detail: own.detail }
+        : { metric: skewTail .metric, value: Math.round(skewTail .ratio * 10) / 10, unit: 'ratio'         ,
+          detail: `Task duration ratio (${skewTail .metric}) is ${Math.round(skewTail .ratio * 10) / 10}×` };
+      const advice = stragglerAdvice(stage, ctx, tail, reported.detail);
       return {
-        type: 'straggler', stageId: stage.id, impactBand, origin: fix.origin,
-        metric: useSpeculativeMetric ? 'speculativeTasks' : 'stragglerShare',
-        value,
-        unit: useSpeculativeMetric ? 'count' : 'pct',
+        type: 'straggler', stageId: stage.id,
+        impactBand: 'info', ...advice.evidence,
+        metric: reported.metric,
+        value: reported.value,
+        unit: reported.unit,
         speculativeTasks: stage.speculativeTasks ?? 0,
         stragglerCount: stage.stragglerCount ?? 0,
-        confidence: useSpeculativeMetric
-          ? stragglerConfidence(speculativeShare, thresholds.warnPct, thresholds.critPct)
-          : stragglerConfidence(stragglerShare, thresholds.shareWarn, thresholds.critPct),
+        confidence: own?.confidence ?? skewConfidence(skewTail .ratio, skew.ratioWarn),
         validationRequired: `Warning needs at least ${shareLabel(thresholds.floorPctWarn)} of run time at stake, critical ${shareLabel(thresholds.floorPctCrit)}.`,
-        recommendation: `${detail}: rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if uneven data is the cause, ${fix.text}.`,
-        remediation: fix.remediation,
+        recommendation: advice.recommendation,
+        remediation: advice.remediation,
       };
     },
     estimate(finding, ctx)                        {
@@ -1864,7 +2368,7 @@ export const DETECTORS = [
     emits: ['speculationWaste'],
     docAnchor: '#bottleneck-speculation-waste',
     thresholds: { minWasted: 5, minWasteMs: 60000 },
-    detect(stage, _ctx, thresholds)                 {
+    detect(stage, ctx, thresholds)                 {
       const wasted = stage.speculationWastedAttempts ?? 0;
       const wastedMs = stage.speculationWasteMs ?? 0;
       if (wasted < thresholds.minWasted || wastedMs < thresholds.minWasteMs) return null;
@@ -1873,7 +2377,7 @@ export const DETECTORS = [
         impactBand: 'warning',
         metric: 'speculationWasteMs', value: wastedMs,
         confidence: speculationWasteConfidence(wastedMs, thresholds.minWasteMs),
-        recommendation: `Speculative execution discarded ${Math.round(wastedMs / 1000)}s of executor time in this stage: if task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.`,
+        recommendation: `Speculative execution discarded ${Math.round(wastedMs / 1000)}s of executor time in this stage: if task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile${speculationRuleNote(ctx.app)}.`,
         remediation: [increaseConf('spark.speculation.multiplier'), increaseConf('spark.speculation.quantile')],
       };
     },
@@ -1921,8 +2425,7 @@ export const DETECTORS = [
     emits: ['tinyTask'],
     docAnchor: '#bottleneck-tiny-tasks',
     // stageFloorPct: the tiered detectors' 0.5% runtime floor. Coalescing can't save more than the
-    // stage's own duration, so on a shorter stage every finding graded info: on the 14 real logs
-    // 132 of 151 tinyTask findings. The tasks are still tiny there; the floor is why they're dropped.
+    // stage's own duration, so on a shorter stage every finding would grade info.
     thresholds: { minTasks: 100, maxP50: 500, maxP95: 1000, stageFloorPct: 0.005 },
     detect(stage, ctx, thresholds)                 {
       if (stage.taskCount < thresholds.minTasks) return null;
@@ -1930,14 +2433,12 @@ export const DETECTORS = [
       if (stage.taskDurationP50 > thresholds.maxP50 || stage.taskDurationP95 > thresholds.maxP95) return null;
       const coalesceTo = Math.max(1, Math.round(stage.taskCount / 10));
       const reads = stageReads(stage);
-      const fix = reads === 'shuffle'
-        ? `lower spark.sql.shuffle.partitions or .coalesce(${coalesceTo})`
-        : `.coalesce(${coalesceTo})`;
+      const fix = reads === 'shuffle' ? tinyShuffleFix(stage, ctx, coalesceTo) : { text: `.coalesce(${coalesceTo})`, remediation: [] };
       return {
         type: 'tinyTask', stageId: stage.id, impactBand: 'info', reads,
         metric: 'taskDurationP50', value: Math.round(stage.taskDurationP50),
-        recommendation: `Many small tasks (${stage.taskCount}, P50 ${Math.round(stage.taskDurationP50)}ms): scheduler overhead may dominate. Try ${fix}.`,
-        remediation: reads === 'shuffle' ? [decreaseConf('spark.sql.shuffle.partitions')] : [],
+        recommendation: `Many small tasks (${stage.taskCount}, P50 ${Math.round(stage.taskDurationP50)}ms): scheduler overhead may dominate. Try ${fix.text}.`,
+        remediation: fix.remediation,
       };
     },
     estimate(finding, ctx)                        {
@@ -1990,10 +2491,7 @@ export const DETECTORS = [
       // No stage ever recorded a submission timestamp: no basis to measure a startup gap against.
       if (!Number.isFinite(firstStageSubmitted)) return null;
       // The wait is from the first runnable stage to the first executor, not from app start: the
-      // driver's own startup before its first job (36-47s on every real log, whatever the
-      // executors did) isn't something executors could shorten. On the 9 real logs that fired,
-      // the first executor arrived 146-192s after the first stage on two whose old gap read ~40s,
-      // and 2s after it on one the old gap flagged critical at 42s.
+      // driver's own startup before its first job isn't something executors could shorten.
       // An executor added before the first stage only counts if it was still alive at submission:
       // with dynamic allocation scaling to zero, early executors can idle out before any job runs.
       const removedAt = new Map                ();
@@ -2036,7 +2534,6 @@ export const DETECTORS = [
     thresholds: { minUtil: 0.60 },
     detect(ctx, thresholds)                 {
       const { app, executorsAdded, executorsRemoved, runAggregates } = ctx;
-      // Nullish (not falsy) check: a literal startTime:0 must not be treated as "missing".
       if (!app || executorsAdded.length === 0 || app.startTime == null || app.endTime == null) return null;
       const appDuration = app.endTime - app.startTime;
       if (appDuration <= 0) return null;
@@ -2052,13 +2549,12 @@ export const DETECTORS = [
       // Busy core-time (from the whole-run core-time-series, same signal memoryUtilization's
       // idleCores variant already uses), not executor lifetime: an executor that exists for the
       // whole run but sits fully idle must not score as 100% used. Missing runAggregates (older
-      // callers, synthetic fixtures) reads as 0 busy time rather than falling back to the
-      // lifetime-based measure this replaces.
+      // callers, synthetic fixtures) reads as 0 busy time.
       const busyCoreMs = runAggregates?.busyCoreMs ?? 0;
       const utilization = busyCoreMs / capacityCoreMs;
       if (utilization >= thresholds.minUtil) return null;
 
-      // CPU-time-based utilization (sparkMeasure): metric only, no threshold.
+      // CPU-time-based utilization: metric only, no threshold.
       // Null when no stage recorded CPU time (older Spark), the same rule as the CLI metrics block.
       const cpuMs = totalExecutorCpuMs(ctx.stages.values());
       const cpuUtilizationPct = cpuMs == null ? null : Math.round((cpuMs / capacityCoreMs) * 100);
@@ -2098,26 +2594,20 @@ export const DETECTORS = [
     emits: ['memoryUtilization'],
     docAnchor: '#bottleneck-memory-utilization',
     thresholds: {
-      idleCoreWarn: 0.50,          // WastedCoresAlertsReducer
-      bandTooSmall: 0.95,          // MemoryAlertsReducer: used/allocated
+      idleCoreWarn: 0.50,          // idle share of allocated core time above which idleCores fires
       bandTooHigh: 0.70,           // below this => over-provisioned (cost signal)
-      wasteBufferMultiplier: 1.5,  // UNVERIFIED
+      wasteBufferMultiplier: 1.5,  // wasted memory-time must exceed used by this multiple
     },
     detect(ctx, thresholds)            {
       const { app, executorsAdded, executorsRemoved, runAggregates, stages } = ctx;
       const out            = [];
-      // Nullish (not falsy) check: a literal startTime:0 must not be treated as "missing".
       if (app?.startTime == null || app?.endTime == null) return out;
       const appDurationMs = app.endTime - app.startTime;
       if (appDurationMs <= 0) return out;
 
-      // Peak concurrent executors/cores (not executorsAdded.length/computeTotalCores): a
-      // cumulative sum or count double-counts a churned-through executor against its replacement's
-      // (spot preemption, dynamicAllocation replacement), inflating idle-rate and waste-model figures.
-      const peakExecutors = computePeakConcurrentExecutorCount(executorsAdded, executorsRemoved);
       const allocatedMB = app.resources?.executor?.memoryMB ?? null;
 
-      // ── 1a idle-cores rate ────────────────────────────────────────────────
+      // Idle-cores rate.
       const busyCoreMs = runAggregates?.busyCoreMs;
       // Allocated core-time, as for `utilization`, so the two findings of one idle condition agree.
       const allocationInput = { app, stages, executors: { added: executorsAdded, removed: executorsRemoved } };
@@ -2141,63 +2631,63 @@ export const DETECTORS = [
         }
       }
 
-      // ── 1b memory bands (executor only; driver half dropped since there is no driver metric) ─
-      // Peak heap per executor = max jvmHeapMemory across all stages' executorMetrics.
-      const peakHeapByExec = new Map                ();
-      for (const s of stages.values()) {
-        const em = s.executorMetrics;
-        if (!(em instanceof Map)) continue;
-        for (const [execId, m] of em) {
-          const heap = m?.jvmHeapMemory ?? 0;
-          if (heap > (peakHeapByExec.get(execId) ?? 0)) peakHeapByExec.set(execId, heap);
-        }
-      }
+      // Memory band (executor only: there is no driver heap metric).
+      // spark.executor.memory is one setting for every executor, so the judgement is made once, on
+      // the executor with the highest sampled heap peak. A near-capacity peak is not reported:
+      // JVMHeapMemory counts uncollected garbage, so a peak near -Xmx is normal JVM behaviour.
+      const peakHeapByExec = executorHeapPeaks(ctx);
       if (peakHeapByExec.size === 0) {
-        const key = 'spark.eventLog.logStageExecutorMetrics';
-        const fix = switchFix(loggedAs(app, key, true), key, true,
-          `Per-executor memory usage requires ${key}=true: not enabled for this run.`,
-          'Per-executor memory usage is missing from this log even though executor metrics logging is on for this run.');
+        // Spark 3.0+ writes the executor's metric peaks on every TaskEnd, but it samples them at the
+        // executor heartbeat (spark.executor.metrics.pollingInterval defaults to 0), so a task that
+        // finishes between two heartbeats reports zeros, as does every task in local mode.
+        // spark.eventLog.logStageExecutorMetrics logs the per-stage peaks the driver gathers.
+        const stageMetrics = switchFix(loggedAs(app, LOG_STAGE_EXECUTOR_METRICS_KEY, true), LOG_STAGE_EXECUTOR_METRICS_KEY, true,
+          ` To measure it, set ${LOG_STAGE_EXECUTOR_METRICS_KEY}=true to record the peaks per stage.`,
+          ` ${LOG_STAGE_EXECUTOR_METRICS_KEY} is already on, so no executor ran long enough to be sampled.`);
         out.push({
           type: 'memoryUtilization', variant: 'memoryBand', stageId: null,
           impactBand: 'info', metric: 'memoryBand', dataUnavailable: true,
-          recommendation: fix.text,
-          remediation: fix.remediation,
+          recommendation: `Executor heap peaks are missing from this log: Spark samples them at the executor heartbeat, so tasks shorter than the heartbeat interval report zeros (and local mode reports zeros throughout), and executor memory sizing was not measured.${stageMetrics.text}`,
+          remediation: stageMetrics.remediation,
         });
       } else if (allocatedMB != null && allocatedMB > 0) {
         const allocatedBytes = allocatedMB * 1024 * 1024;
-        for (const [execId, heap] of peakHeapByExec) {
-          const ratio = heap / allocatedBytes;
-          // The two bands are opposite signals: an explicit `rule` discriminator lets consumers
-          // tell OOM-risk from over-provisioning without re-deriving the ratio.
-          if (ratio > thresholds.bandTooSmall) {
-            out.push({
-              type: 'memoryUtilization', variant: 'memoryBand', rule: 'heapNearCapacity',
-              stageId: null, executorId: execId,
-              impactBand: 'warning', metric: 'heapUsedRatio', value: Math.round(ratio * 100),
-              recommendation: `Executor ${execId} peaked at ${Math.round(ratio * 100)}% of allocated heap: memory may be too small; raise spark.executor.memory to avoid OOM/spill.`,
-              remediation: [increaseConf('spark.executor.memory')],
-            });
-          } else if (ratio < thresholds.bandTooHigh) {
-            out.push({
-              type: 'memoryUtilization', variant: 'memoryBand', rule: 'heapOverProvisioned',
-              stageId: null, executorId: execId,
-              impactBand: 'info', metric: 'heapUsedRatio', value: Math.round(ratio * 100),
-              // Absolute figures behind the rounded ratio, for the estimator's
-              // unused-memory-over-time model.
-              allocatedBytes, heap, appDurationMs,
-              recommendation: `Executor ${execId} used only ${Math.round(ratio * 100)}% of allocated heap: memory may be over-provisioned; consider reducing spark.executor.memory for cost savings.`,
-              remediation: [decreaseConf('spark.executor.memory')],
-            });
-          }
+        let execId = '';
+        let heap = 0;
+        for (const [id, peak] of peakHeapByExec) {
+          if (peak > heap) { heap = peak; execId = id; }
+        }
+        const ratio = heap / allocatedBytes;
+        if (ratio < thresholds.bandTooHigh) {
+          const pct = Math.round(ratio * 100);
+          const executorCount = peakHeapByExec.size;
+          out.push({
+            type: 'memoryUtilization', variant: 'memoryBand', rule: 'heapOverProvisioned',
+            stageId: null, executorId: execId,
+            impactBand: 'info', metric: 'heapUsedRatio', value: pct,
+            // Absolute figures behind the rounded ratio, for the estimator's unused-memory model:
+            // the unused heap is held by every executor for as long as it is alive.
+            allocatedBytes, heap, executorCount,
+            executorSeconds: computeAllocation({ app, stages, executors: { added: executorsAdded, removed: executorsRemoved } }).executorSeconds
+              ?? executorCount * (appDurationMs / 1000),
+            recommendation: `The busiest of ${executorCount} sampled executors peaked at ${pct}% of allocated heap (a lower bound: Spark samples at executor heartbeat): memory may be over-provisioned; consider reducing spark.executor.memory for cost savings.`,
+            remediation: [decreaseConf('spark.executor.memory')],
+          });
         }
       }
 
-      // ── 1c Spark Memory Limit waste model (UNVERIFIED buffer) ─
-      if (allocatedMB != null && peakExecutors > 0) {
-        const allocatedMBSeconds = peakExecutors * allocatedMB * (appDurationMs / 1000);
+      // Spark Memory Limit waste model.
+      // executorRunTime sums task time across cores, so dividing by the cores per executor gives
+      // executor-seconds: the same unit as the allocated executor-seconds it is compared with.
+      // The allocated side is the executors' alive time (computeAllocation), not the peak executor
+      // count held for the whole run: under dynamic allocation the two differ by peak over mean.
+      const allocation = computeAllocation(allocationInput);
+      const executorCores = app.resources?.executor?.cores ?? allocation.executorCores;
+      if (allocatedMB != null && allocation.executorSeconds != null && allocation.executorSeconds > 0 && executorCores != null && executorCores > 0) {
+        const allocatedMBSeconds = allocatedMB * allocation.executorSeconds;
         let usedRunTimeMs = 0;
         for (const s of stages.values()) usedRunTimeMs += s.executorRunTime ?? 0;
-        const usedMBSeconds = allocatedMB * (usedRunTimeMs / 1000);
+        const usedMBSeconds = allocatedMB * (usedRunTimeMs / 1000 / executorCores);
         const wastedMBSeconds = allocatedMBSeconds - usedMBSeconds;
         if (wastedMBSeconds > thresholds.wasteBufferMultiplier * usedMBSeconds) {
           const value = Math.round(wastedMBSeconds);
@@ -2229,15 +2719,15 @@ export const DETECTORS = [
         }
         return costOnly('modeled');
       }
-      // Only the over-provisioned band is a waste; the near-capacity band is an OOM-risk signal with
-      // no magnitude, and the dataUnavailable shape has no inputs: both stay informational.
+      // Only the over-provisioned band is a waste; the dataUnavailable shape has no inputs and
+      // stays informational.
       if (finding.variant === 'memoryBand' && finding.rule === 'heapOverProvisioned') {
         const allocatedBytes = finding.allocatedBytes                      ;
         const heap = finding.heap                      ;
-        const appDurationMs = finding.appDurationMs                      ;
-        if (allocatedBytes != null && heap != null && appDurationMs != null) {
+        const executorSeconds = finding.executorSeconds                      ;
+        if (allocatedBytes != null && heap != null && executorSeconds != null) {
           const unusedMB = (allocatedBytes - heap) / (1024 * 1024);
-          const wastedMBSeconds = unusedMB * (appDurationMs / 1000);
+          const wastedMBSeconds = unusedMB * executorSeconds;
           return costOnly('modeled', { value: wastedMBSeconds, unit: 'mbSeconds' });
         }
       }
@@ -2264,7 +2754,7 @@ export const DETECTORS = [
       const out            = [];
       let persistedRddCount = 0;
       // With block-update logging on, zero rdd_* updates means nothing was ever cached, not a gap.
-      const blockUpdatesLogged = String(ctx.app?.config?.['spark.eventLog.logBlockUpdates.enabled']).toLowerCase() === 'true';
+      const blockUpdatesLogged = loggedAs(ctx.app, 'spark.eventLog.logBlockUpdates.enabled', true);
       // Spark before 2.3 has no block-update logging and writes RDD Info's cache figures only on
       // StageCompleted, which isn't read: the caveat's advice doesn't apply there. A log with no
       // version is pre-1.3 (no SparkListenerLogStart); every 2.3+ log records one.
@@ -2311,7 +2801,10 @@ export const DETECTORS = [
       const uncachedBytes = numCachedPartitions > 0 ? (cachedBytes / numCachedPartitions) * numUncachedPartitions : 0;
       const uncachedOrSpilledBytes = uncachedBytes + diskSize;
       const wasteMs = (uncachedOrSpilledBytes / RE_READ_THROUGHPUT_BPS) * 1000;
-      return costOnly('modeled', { value: wasteMs, unit: 'ms' });
+      return withAssumptions(
+        costOnly('modeled', { value: wasteMs, unit: 'ms' }),
+        `assumes ${Math.round(RE_READ_THROUGHPUT_BPS / 1e6)} MB/s per reading task, summed as task time and not divided over executors`,
+      );
     },
   }),
   defineAppDetector({
@@ -2581,8 +3074,13 @@ export const DETECTORS = [
     },
     estimate(finding)                        {
       const totalReadBytes = (finding.totalReadBytes                      ) ?? 0;
-      const wasteMs = (totalReadBytes / RE_READ_THROUGHPUT_BPS) * 1000;
-      return costOnly('modeled', { value: wasteMs, unit: 'ms' });
+      // The measured scan volume and the query count, not a time: the bytes are cumulative task
+      // input across queries, and a re-read cost depends on how much of it a cache would serve.
+      const queries = (finding.executionIds                        )?.length ?? 0;
+      return withAssumptions(
+        costOnly('measured', { value: totalReadBytes, unit: 'bytes', reread: true }),
+        `counts the bytes ${queries} ${queries === 1 ? 'query' : 'queries'} scanned from the same relation, matched by plan; a cache would still read it once`,
+      );
     },
   }),
   defineAppDetector({
@@ -2625,43 +3123,18 @@ export const DETECTORS = [
       return costOnly('modeled', { value: coreHoursIsh, unit: 'coreHours' });
     },
   }),
-  // ── Config-sanity entries (scope:'config', inScorecard:false) ────────────────
-  defineConfigDetector({
-    type: 'configAudit', order: 120, fixEffort: 'config', version: 1, inScorecard: false,
-    emits: ['configAudit'],
-    docAnchor: '#config-shuffle-service', thresholds: {}, property: 'spark.shuffle.service.enabled',
-    detect(target)                 {
-      const res = target.app?.resources ?? null;
-      if (res?.dynamicAllocationEnabled === true && res?.shuffleServiceEnabled === false) {
-        return {
-          type: 'configAudit', property: 'spark.shuffle.service.enabled',
-          impactBand: 'warning', metric: 'config', valueText: 'false',
-          recommendation: 'Dynamic allocation is on but the external shuffle service is off: set spark.shuffle.service.enabled=true so shuffle data survives executor removal.',
-          remediation: setConfUnlessLogged(target.app, 'spark.shuffle.service.enabled', true),
-        };
-      }
-      return null;
-    },
-    estimate: noWasteModel,
-  }),
   defineConfigDetector({
     type: 'configAudit', order: 121, fixEffort: 'config', version: 1, inScorecard: false,
     emits: ['configAudit'],
     docAnchor: '#config-autoscale-bounds', thresholds: {}, property: 'spark.dynamicAllocation.maxExecutors',
     detect(target)                 {
-      const app = target.app; const config = app?.config ?? {}; const res = app?.resources ?? null;
+      const app = target.app; const res = app?.resources ?? null;
       if (res?.dynamicAllocationEnabled !== true) return null;
-      const minN = config['spark.dynamicAllocation.minExecutors'] != null ? parseInt(config['spark.dynamicAllocation.minExecutors'], 10) : null;
-      const maxN = config['spark.dynamicAllocation.maxExecutors'] != null ? parseInt(config['spark.dynamicAllocation.maxExecutors'], 10) : null;
-      if (minN != null && maxN != null && minN > maxN) {
-        return {
-          type: 'configAudit', property: 'spark.dynamicAllocation.minExecutors',
-          impactBand: 'critical', metric: 'config', valueText: `${minN} > ${maxN}`,
-          recommendation: `spark.dynamicAllocation.minExecutors (${minN}) exceeds maxExecutors (${maxN}): set min ≤ max.`,
-          remediation: [decreaseConf('spark.dynamicAllocation.minExecutors', maxN)],
-        };
-      }
-      if (maxN == null) {
+      // minExecutors > maxExecutors is not audited: ExecutorAllocationManager.validateSettings throws on it
+      // at startup (v3.5.0 lines 196-199), so no run that wrote an event log has inverted bounds.
+      // Unbounded is the effective maxExecutors being Spark's default (Int.MaxValue) or that value set.
+      const maxN = Number.parseInt(effectiveConf(app, 'spark.dynamicAllocation.maxExecutors') ?? '', 10);
+      if (!Number.isFinite(maxN) || maxN >= UNBOUNDED_EXECUTORS) {
         return {
           type: 'configAudit', property: 'spark.dynamicAllocation.maxExecutors',
           impactBand: 'info', metric: 'config', valueText: '(unset)',
@@ -2680,7 +3153,13 @@ export const DETECTORS = [
     detect(target)                 {
       const app = target.app; const config = app?.config ?? {}; const res = app?.resources ?? null;
       if (Object.keys(config).length === 0) return null;
-      const ser = res?.serializer ?? config['spark.serializer'] ?? null;
+      // spark.serializer does not serialize SQL rows (shuffles and caches use UnsafeRow and the columnar
+      // cache serializer), so the note only applies to a run that ran stages outside any SQL execution.
+      // Without the stages there is no evidence of RDD work and no claim.
+      if (!hasStageOutsideSql(target.stages)) return null;
+      // The serializer the run set; Spark's own default (JavaSerializer) is reported as the default.
+      const effective = effectiveSparkConf({ sparkVersion: app?.sparkVersion, properties: config }, 'spark.serializer');
+      const ser = res?.serializer ?? (effective?.source === 'default' ? null : effective?.value ?? null);
       const isKryo = typeof ser === 'string' && /kryo/i.test(ser);
       if (isKryo) return null;
       return {
@@ -2701,26 +3180,25 @@ export const DETECTORS = [
       const memMB = res?.executor?.memoryMB ?? null;
       const ovMB = res?.executor?.memoryOverheadMB ?? null;
       if (memMB == null || ovMB == null) return null;
-      const floor = Math.max(thresholds.floorMB, Math.round(memMB * thresholds.floorPct));
+      const { minMB, factor } = defaultOverheadSettings(target.app, thresholds);
+      const floor = Math.max(minMB, Math.round(memMB * factor));
       if (ovMB >= floor) return null;
       return {
         type: 'configAudit', property: 'spark.executor.memoryOverhead',
         impactBand: 'info', metric: 'config', valueText: `${ovMB} MiB`,
-        recommendation: `Executor memoryOverhead (${ovMB} MiB) is below Spark's default floor of ${floor} MiB (max of 384 MiB or 10% of executor memory): raise it to avoid off-heap OOM-kills.`,
+        recommendation: `Executor memoryOverhead (${ovMB} MiB) is below the ${floor} MiB Spark would have defaulted to (max of ${minMB} MiB or ${Math.round(factor * 1000) / 10}% of executor memory): raise it to avoid off-heap OOM-kills.`,
         remediation: [increaseConf('spark.executor.memoryOverhead', `${floor}m`)],
       };
     },
     estimate: noWasteModel,
   }),
-  // ── Plan-metric entries (scope:'sql') ────────────────────────────────────
   defineSqlDetector({
     type: 'duplicatePlanSubtree', order: 130, fixEffort: 'code', version: 2,
     emits: ['duplicatePlanSubtree'],
     docAnchor: '#bottleneck-duplicate-plan-subtree',
     // stageFloorPct: the 0.5% runtime floor. The claim counts at most each linked stage's own
     // task-active time, so a repeat whose stages together lasted less than this share of the run
-    // graded info: 340 of 545 findings on the 14 real logs. The repeat is still in the plan; the
-    // floor is why they're dropped. A repeat with no linked stage time is kept.
+    // would grade info. A repeat with no linked stage time is kept.
     thresholds: { minSubtreeSize: 3, minOccurrences: 2, stageFloorPct: 0.005 },
     detect(sqlExec, ctx, thresholds)                   {
       if (!sqlExec.planTree) return null;
@@ -2743,16 +3221,15 @@ export const DETECTORS = [
         if (runMs != null && stagesMs > 0 && stagesMs < runMs * thresholds.stageFloorPct) return null;
         const occurrencesIdentical = occurrencesHaveIdenticalDetails(g.nodes);
         const stageShares = stageOperatorShares(nodes, operatorsByStage);
-        // resolvePlanTree always sets id; safe downstream of it.
+        // resolvePlanTree always sets id, so the `!` here and below is safe.
         const planNodeIds = nodes.map((n) => n.id ).filter(Boolean);
         const detail = duplicateSubtreeDetail({ ...g, value: g.occurrences });
         const differing = occurrencesIdentical ? '' : ` ${DUPLICATE_SUBTREE_DIFFERING_NOTE}`;
         return {
           type: 'duplicatePlanSubtree', executionId: sqlExec.id, stageIds, planNodeIds,
           stageShares, occurrencesIdentical,
-          // Fixed fallback: overwritten by deriveImpactBand when this finding gets a real
-          // wallClock estimate. Repeats with differing details get no estimate (nothing is
-          // known to be recomputed), and neither does a subtree with no stage of its own.
+          // Repeats with differing details get no estimate (nothing is known to be recomputed),
+          // and neither does a subtree with no stage of its own, so this fallback band stands.
           impactBand: occurrencesIdentical && Object.keys(stageShares).length > 0 ? 'warning' : 'info',
           metric: 'subtreeOccurrences', value: g.occurrences,
           rootName: g.rootName, subtreeSize: g.subtreeSize, sampleRelation: g.sampleRelation,
@@ -2772,7 +3249,7 @@ export const DETECTORS = [
       // detect() reports subtreeOccurrences >= 2. Only repeats past the first are redundant:
       // computing the subtree once is real work, so waste is (occurrences-1)/occurrences of the stages' time.
       const occurrences = typeof finding.value === 'number' ? finding.value : 0;
-      // Defensive only: minOccurrences guarantees occurrences >= 2 on real data; a malformed-value fallback.
+      // One occurrence repeats nothing: reached only with minOccurrences tuned below 2 or a malformed value.
       if (occurrences < 2) return costOnly('none');
       // Same-shaped repeats whose details differ compute different data: nothing is known to be
       // recomputed, so there is no time to claim.
@@ -2788,8 +3265,8 @@ export const DETECTORS = [
         const s = ctx.stages.get(id);
         const share = shares ? (shares[id] ?? 0) : 1;
         if (s && share > 0) {
-          // Time with tasks running, not submit-to-complete: a stage left waiting for cores
-          // (2491s open, 60s of tasks on a real log) isn't recomputing anything while it waits.
+          // Time with tasks running, not submit-to-complete: a stage waiting for cores isn't
+          // recomputing anything.
           const activeMs = s.taskActiveMs ?? Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0));
           wasteMsByStage.set(id, activeMs * share * redundantFraction);
         }
@@ -2847,8 +3324,7 @@ export const DETECTORS = [
     estimate(finding, ctx)                        {
       const fileMs = ((finding.fileCount                      ) ?? 0) * FILE_OPEN_OVERHEAD_MS;
       // A read's files are opened by the scan's tasks, in parallel: spread the per-file cost over
-      // the most tasks its stages ran at once (91344 files x 10ms is 913s, claimed against a 117s
-      // stage that ran 314 tasks at once). A write keeps the serial sum: the job commit moves each
+      // the most tasks its stages ran at once. A write keeps the serial sum: the job commit moves each
       // output file on the driver, one after another.
       const stageIds = finding.stageIds                        ;
       let slots = 1;
@@ -2861,50 +3337,72 @@ export const DETECTORS = [
   defineSqlDetector({
     // Entry-level type is an identifier only; it never appears on an emitted finding. Findings
     // carry 'underBroadcast'/'overBroadcast' since one shared plan-walk covers both
-    // opposite-direction rules (JoinToBroadcastAlert / BroadcastTooLargeAlert).
-    type: 'broadcastSizing', order: 132, fixEffort: 'config', version: 2,
+    // opposite-direction rules.
+    type: 'broadcastSizing', order: 132, fixEffort: 'config', version: 3,
     // Listed over-first: the two share order 132, and this list order is their display tie-break.
     emits: ['overBroadcast', 'underBroadcast'],
     docAnchor: '#bottleneck-broadcast-sizing',
     thresholds: {
-      broadcastTiers: [10 * MB, 100 * MB, GB, 5 * GB],
-      comparisonTiers: [10 * GB, 300 * GB, TB],
+      broadcastTiers: [10 * MB, 100 * MB, GB],
+      comparisonTiers: [10 * GB, 300 * GB],
       overBroadcastBytes: GB,
+      minSmallerSideBytes: MB,
     },
     detect(sqlExec, ctx, thresholds)                   {
       if (!sqlExec.planTree) return null;
-      const { broadcastTiers, comparisonTiers, overBroadcastBytes } = thresholds;
+      const { broadcastTiers, comparisonTiers, overBroadcastBytes, minSmallerSideBytes } = thresholds;
       const fallbackStageIds = stageIdsForSqlExec(sqlExec.id, ctx.stages);
+      const app = queryApp(ctx.app, sqlExec.modifiedConfigs);
+      const thresholdsFor = (adaptive         ) => {
+        const key = broadcastThresholdKey(app, adaptive);
+        // A broadcast planned up front is admitted by the static threshold, one AQE converted at runtime
+        // by the adaptive one, so an existing broadcast is judged against each that applied.
+        const admittingKeys = key === BROADCAST_THRESHOLD_KEY ? [key] : [BROADCAST_THRESHOLD_KEY, key];
+        return {
+          key, threshold: effectiveBroadcastThreshold(app, key),
+          admittingThresholds: admittingKeys.map((k) => ({ key: k, threshold: effectiveBroadcastThreshold(app, k) })),
+        };
+      };
+      const staticThresholds = thresholdsFor(false);
+      const adaptiveThresholds = thresholdsFor(true);
+      // AdaptiveSparkPlan need not be the root: Spark 3.2-3.3 wraps only a write command's child.
+      const underAdaptive = new Set          ();
       const out            = [];
-      walkPlanTree(sqlExec.planTree, (node) => {
-        if (node.name === 'SortMergeJoin' && (node.children ?? []).length === 2) {
-          const [childA, childB] = node.children;
-          const a = sumBoundarySize(childA);
-          const b = sumBoundarySize(childB);
-          const smaller = Math.min(a, b);
-          const larger = Math.max(a, b);
-          if (smaller > 0) {
-            const fires = smaller < broadcastTiers[0]
-              || (smaller < broadcastTiers[1] && larger > comparisonTiers[0])
-              || (smaller < broadcastTiers[2] && larger > comparisonTiers[1])
-              || (smaller < broadcastTiers[3] && larger > comparisonTiers[2]);
-            if (fires) {
-              const contributors = [...boundarySizeContributors(childA), ...boundarySizeContributors(childB)];
-              // A threshold that already admits the smaller side cannot be what stopped the broadcast.
-              const threshold = effectiveBroadcastThreshold(ctx.app);
-              const broadcastThreshold                     = threshold != null && threshold < 0 ? 'disabled'
-                : threshold != null && threshold > smaller ? 'notLimiting' : 'limits';
-              const raiseAdvice = 'Consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.';
+      walkPlanTree(sqlExec.planTree, (node, parent) => {
+        if (node.name === 'AdaptiveSparkPlan' || (parent != null && underAdaptive.has(parent))) underAdaptive.add(node);
+        const adaptive = underAdaptive.has(node);
+        const { key, threshold, admittingThresholds } = adaptive ? adaptiveThresholds : staticThresholds;
+        const candidate = node.name === 'SortMergeJoin' ? broadcastCandidate(node) : null;
+        if (candidate) {
+          const buildBytes = shuffleBytes(candidate.build);
+          const otherBytes = shuffleBytes(candidate.stream);
+          // Below the floor a broadcast saves nothing measurable, nor does broadcasting the larger side;
+          // above the over-broadcast limit it would be flagged there instead.
+          if (buildBytes >= minSmallerSideBytes && buildBytes <= overBroadcastBytes && buildBytes <= otherBytes) {
+            const broadcastThreshold                     = threshold != null && threshold < 0 ? 'disabled'
+              : threshold != null && threshold >= buildBytes ? 'notLimiting' : 'limits';
+            const withinTiers = buildBytes < broadcastTiers[0]
+              || (buildBytes < broadcastTiers[1] && otherBytes > comparisonTiers[0])
+              || (buildBytes < broadcastTiers[2] && otherBytes > comparisonTiers[1]);
+            if (broadcastThreshold === 'notLimiting' || withinTiers) {
+              const contributors = [candidate.build, candidate.stream];
+              const { joinType, buildSide } = candidate;
+              const subject = `The ${buildSide} input to this ${joinType} Sort Merge Join (${formatBytes(buildBytes)})`;
+              const otherSide = ` (the other side is ${formatBytes(otherBytes)})`;
               out.push({
                 type: 'underBroadcast', executionId: sqlExec.id, stageIds: unionStageIds(contributors, fallbackStageIds),
-                // resolvePlanTree always sets id; safe downstream of it.
                 planNodeIds: contributors.map((n) => n.id ).filter(Boolean),
-                impactBand: 'info', metric: 'smallerSideBytes', value: smaller,
-                largerSideBytes: larger, broadcastThreshold,
+                impactBand: 'info', metric: 'smallerSideBytes',
+                value: buildBytes, largerSideBytes: otherBytes,
+                joinType, buildSide, broadcastThreshold,
                 recommendation: broadcastThreshold === 'notLimiting'
-                  ? `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is under the effective spark.sql.autoBroadcastJoinThreshold (${formatBytes(threshold )}) yet was not broadcast (the larger side is ${formatBytes(larger)}), so the threshold is not what stopped it: missing table statistics or a join type that cannot broadcast usually is. Consider a broadcast() hint or collecting statistics (ANALYZE TABLE).`
-                  : `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is well under the broadcast threshold relative to the larger side (${formatBytes(larger)}): this could have been a broadcast join. ${raiseAdvice}`,
-                remediation: broadcastThreshold === 'notLimiting' ? [] : [increaseConf('spark.sql.autoBroadcastJoinThreshold')],
+                  ? `${subject} is under the effective ${key} (${formatBytes(threshold )}) yet was not broadcast${otherSide}, so the threshold is not what stopped it: ${adaptive
+                    ? `a join hint, a shuffle that had already run, or a low share of non-empty partitions (below ${NON_EMPTY_PARTITION_RATIO_KEY}) usually is. Consider a broadcast() hint or lowering ${NON_EMPTY_PARTITION_RATIO_KEY}.`
+                    : 'a join hint, missing table statistics, or a shuffle that had already run usually is. Consider a broadcast() hint or collecting statistics (ANALYZE TABLE).'}`
+                  : broadcastThreshold === 'disabled'
+                    ? `${subject} was not broadcast${otherSide} because automatic broadcast is disabled (${key}=-1): this could have been a broadcast join. Set ${key} to a positive size or add a broadcast() hint.`
+                    : `${subject} is over the effective ${key} (${threshold == null ? 'not a size' : formatBytes(threshold)}) but small enough to broadcast${otherSide}: this could have been a broadcast join. Consider a broadcast() hint or raising ${key}.`,
+                remediation: broadcastThreshold === 'notLimiting' ? [] : [increaseConf(key)],
               });
             }
           }
@@ -2912,27 +3410,27 @@ export const DETECTORS = [
         if (isBroadcastExchangeNode(node.name)) {
           const m = (node.metrics ?? []).find(x => x.name === 'data size');
           if (m && m.value > overBroadcastBytes) {
-            // BroadcastExchange's own metrics are driver-computed and never on any TaskEnd
-            // (node.stageIds always empty in real data); its child carries the executor-side
-            // metrics, so only the child unions in.
+            // BroadcastExchange's own metrics are driver-computed and never on a TaskEnd, so only
+            // its child, which carries the executor-side metrics, contributes stage ids.
             const child = (node.children ?? [])[0];
-            // A threshold below the broadcast cannot have admitted it, so a hint forced it.
-            const threshold = effectiveBroadcastThreshold(ctx.app);
-            const autoBroadcastOff = threshold != null && threshold < 0;
-            const hintForced = autoBroadcastOff || (threshold != null && m.value > threshold);
+            // A threshold below the broadcast cannot have admitted it: with none admitting it, a hint forced it.
+            const admitted = admittingThresholds.filter((t) => t.threshold == null || t.threshold >= m.value);
+            const autoBroadcastOff = admittingThresholds.every((t) => t.threshold != null && t.threshold < 0);
+            const hintForced = admitted.length === 0;
             const broadcastThreshold                     = autoBroadcastOff ? 'disabled' : hintForced ? 'notLimiting' : 'limits';
+            const below = admittingThresholds.map((t) => `${t.key} (${t.threshold  < 0 ? 'disabled' : formatBytes(t.threshold )})`).join(' and ');
+            const admittedKeys = admitted.map((t) => t.key);
             out.push({
               type: 'overBroadcast', executionId: sqlExec.id,
               stageIds: unionStageIds(child ? [child] : [], fallbackStageIds),
-              // resolvePlanTree always sets id; safe downstream of it.
               planNodeIds: [node.id ].filter(Boolean),
               impactBand: 'warning', metric: 'broadcastBytes', value: m.value, broadcastThreshold,
               recommendation: autoBroadcastOff
                 ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: automatic broadcast is already disabled, so remove the broadcast() hint that forced it.`
                 : hintForced
-                  ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: spark.sql.autoBroadcastJoinThreshold (${formatBytes(threshold )}) is below it, so a broadcast() hint forced it: remove the hint.`
-                  : `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
-              remediation: hintForced ? [] : [decreaseConf('spark.sql.autoBroadcastJoinThreshold')],
+                  ? `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: ${below} ${admittingThresholds.length > 1 ? 'are' : 'is'} below it, so a broadcast() hint forced it: remove the hint.`
+                  : `This broadcast (${formatBytes(m.value)}) exceeds the ${binaryThresholdLabel(overBroadcastBytes)} threshold: check for a misapplied broadcast hint or a misconfigured ${admittedKeys.join(' or ')}.`,
+              remediation: admittedKeys.map((k) => decreaseConf(k)),
             });
           }
         }
@@ -2941,15 +3439,138 @@ export const DETECTORS = [
     },
     estimate(finding, ctx)                        {
       // Both finding types carry bytes as `value`: overBroadcast's broadcastBytes, underBroadcast's
-      // smallerSideBytes (the smaller join side), each priced as one broadcast transfer.
+      // smallerSideBytes (its build side), each priced as one broadcast transfer.
       const wasteMs = (((finding.value                      ) ?? 0) / BROADCAST_BANDWIDTH_BPS) * 1000;
       return stageMappableWasteOrCostOnly(wasteMs, finding.stageIds                        , ctx);
     },
   }),
+  defineSqlDetector({
+    type: 'pythonUdf', order: 134, fixEffort: 'code', version: 1,
+    emits: ['pythonUdf'],
+    docAnchor: '#pyspark',
+    // Unvalidated against a real workload (no corpus or private log runs a Python UDF): set high so
+    // only a stage that moved a lot of data for a long time is flagged.
+    thresholds: { minBytesSent: 64 * MB, minStageMs: 30_000 },
+    detect(sqlExec, ctx, thresholds)                   {
+      if (!sqlExec.planTree) return null;
+      const { minBytesSent, minStageMs } = thresholds;
+      // BatchEvalPython runs a row-at-a-time UDF; ArrowEvalPython (an Arrow-optimized or pandas UDF)
+      // is already the fix. Both Python byte metrics are executor-side; a plan that reports no sent
+      // bytes is skipped.
+      const nodes             = [];
+      let sentBytes = 0;
+      let returnedBytes = 0;
+      let reportsReturned = false;
+      walkPlanTree(sqlExec.planTree, (node) => {
+        if (!isBatchEvalPythonNode(node)) return;
+        const sent = node.metrics?.find((m) => m.name === PYTHON_DATA_SENT)?.value;
+        if (sent == null || !(node.stageIds?.length)) return;
+        nodes.push(node);
+        sentBytes += sent;
+        const returned = node.metrics?.find((m) => m.name === PYTHON_DATA_RETURNED)?.value;
+        if (returned != null) { returnedBytes += returned; reportsReturned = true; }
+      });
+      if (nodes.length === 0 || !(sentBytes >= minBytesSent)) return null;
+      const stageIds = unionStageIds(nodes, []);
+      const stageDurationMs = stageIds.reduce((sum, id) => {
+        const stage = ctx.stages.get(id);
+        return sum + Math.max(0, (stage?.completedAt ?? 0) - (stage?.submittedAt ?? 0));
+      }, 0);
+      if (!(stageDurationMs >= minStageMs)) return null;
+
+      const arrowFix = pythonUdfArrowFix(queryApp(ctx.app, sqlExec.modifiedConfigs));
+      const traffic = `sent ${formatBytes(sentBytes)} to Python workers${reportsReturned ? ` and received ${formatBytes(returnedBytes)} back` : ''}`;
+      return [{
+        type: 'pythonUdf', executionId: sqlExec.id, stageIds, planNodeIds: nodes.flatMap(n => (n.id ? [n.id] : [])),
+        impactBand: 'info',
+        metric: 'dataSentBytes', value: sentBytes,
+        dataSentBytes: sentBytes, dataReturnedBytes: reportsReturned ? returnedBytes : null, stageDurationMs,
+        recommendation: `Row-at-a-time Python UDFs (BatchEvalPython) ${traffic} over ${formatDuration(stageDurationMs)} of stage time: ${arrowFix.text}`,
+        remediation: arrowFix.remediation,
+      }];
+    },
+    estimate: noWasteModel,
+  }),
+  defineSqlDetector({
+    type: 'nestedLoopJoin', order: 133, fixEffort: 'code', version: 1,
+    emits: ['nestedLoopJoin'],
+    docAnchor: '#joins',
+    // minOutputRows: below it the join is cheap whatever its shape. minExpansion: a
+    // BroadcastNestedLoopJoin's output must be this many times its larger input, so a join that
+    // keeps its rows, or a cross join against a one-row side, is not an explosion. The stage time the
+    // finding is graded on decides how much it matters.
+    thresholds: { minOutputRows: 1_000_000, minExpansion: 10 },
+    detect(sqlExec, _ctx, thresholds)                   {
+      if (!sqlExec.planTree) return null;
+      const out            = [];
+      walkPlanTree(sqlExec.planTree, (node) => {
+        const shape = parseNestedLoopJoin(node);
+        if (!shape || node.children.length !== 2) return;
+        const outputRows = ownOutputRows(node);
+        if (outputRows == null || outputRows < thresholds.minOutputRows) return;
+        // A CartesianProduct re-reads its inputs (UnsafeCartesianRDD in joins/CartesianProductExec
+        // computes every left and right partition pair, so each input partition is read once per
+        // partition of the other side), so the executors' input counts exceed the real row counts and cannot be compared with the output. It runs only when neither side can
+        // be broadcast, and its output is the product of the inputs, so size alone qualifies it.
+        const counted = shape.operator === 'CartesianProduct';
+        const leftRows = counted ? null : outputRowsOf(node.children[0]);
+        const rightRows = counted ? null : outputRowsOf(node.children[1]);
+        let expansion                = null;
+        if (!counted) {
+          // Without both input counts "far above both inputs" cannot be shown.
+          if (leftRows == null || rightRows == null) return;
+          expansion = outputRows / Math.max(leftRows, rightRows, 1);
+          if (expansion < thresholds.minExpansion) return;
+        }
+        // Only the stages that run the join: the whole execution's stages would claim time the
+        // join never spent.
+        const stageIds = unionStageIds([node], []);
+        const advice = shape.condition == null
+          ? 'Confirm the cross join is intended, or add a join key so the rows are matched instead of multiplied.'
+          : 'Add an equi-join key so Spark can use a hash or sort-merge join; for a range condition, bucket the range and join on the bucket as well.';
+        const on = shape.condition == null ? '' : ` on ${shape.condition}`;
+        out.push({
+          type: 'nestedLoopJoin', executionId: sqlExec.id, stageIds,
+          planNodeIds: [node.id ].filter(Boolean),
+          impactBand: 'info',
+          metric: 'outputRows', value: outputRows,
+          nodeName: shape.operator, joinType: shape.joinType, condition: shape.condition,
+          outputRows, leftRows, rightRows,
+          confidence: 'medium',
+          validationRequired: 'Row counts are the executors\' measured values; the time graded is the whole stage that runs the join, which also covers its other operators.',
+          recommendation: `${shape.operator}${on} produced ${outputRows.toLocaleString('en-US')} rows${expansion == null
+            ? shape.condition == null
+              ? ', every left row paired with every right row.'
+              // The condition filtered the pairings: the output is the rows it kept, not the product.
+              : ': the pairs its condition kept from every left and right pairing it compared.'
+            : ` from ${leftRows .toLocaleString('en-US')} and ${rightRows .toLocaleString('en-US')} input rows (${Math.round(expansion).toLocaleString('en-US')}x the larger side).`} ${advice}`,
+          remediation: [codeFix(advice)],
+        });
+      });
+      return out.length > 0 ? out : null;
+    },
+    estimate(finding, ctx)                        {
+      const stageIds = finding.stageIds                        ;
+      if (!stageIds || stageIds.length === 0) return costOnly('none');
+      // The join's cost is the time of the stages that run it: a nested-loop join is O(n x m)
+      // inside them. Time with tasks running, not submit-to-complete (a stage waiting for cores
+      // is not looping).
+      const wasteMsByStage = new Map                ();
+      for (const id of stageIds) {
+        const s = ctx.stages.get(id);
+        if (s) wasteMsByStage.set(id, s.taskActiveMs ?? Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0)));
+      }
+      if (wasteMsByStage.size === 0) return costOnly('none');
+      const totalMs = [...wasteMsByStage.values()].reduce((sum, ms) => sum + ms, 0);
+      const rawWaste                 = { value: totalMs, unit: 'ms' };
+      return multiStageImpact([...wasteMsByStage.keys()], wasteMsByStage, ctx, 'modeled', rawWaste)
+        ?? costOnly('modeled', rawWaste);
+    },
+  }),
 ]                                       ;
 
 /** The entry that emits each finding type: the one whose estimate prices it and whose thresholds
- * and order describe it. Several entries can emit one type (the four configAudit audits), and the
+ * and order describe it. Several entries can emit one type (the three configAudit audits), and the
  * first declared wins. */
 export const ENTRY_BY_TYPE                                = (() => {
   const byType = new Map                  ();
@@ -2989,3 +3610,9 @@ export const ENTRY_BY_TYPE                                = (() => {
                                   
                                                                         
   
+
+/** skew's thresholds for a run: its defaults with the user's `skew` overrides merged in. The one
+ * resolution analyze() hands to both skew (through withThresholds) and straggler (DetectorCtx). */
+export function skewThresholdsFor(overrides                                )                 {
+  return mergeThresholds('skew', ENTRY_BY_TYPE.get('skew') .thresholds, overrides?.skew ?? {})                             ;
+}