sparkforensics-mcp 0.2.0 → 0.2.3

package/vendor-core/detectors.js

@@ -3,8 +3,9 @@ import { scanRelationId } from './plan-summary.js';
 import { computePeakConcurrentCores, computePeakConcurrentExecutorCount } from './core-count.js';
 import { walkPlanTree } from './plan-tree-walk.js';
 import { computeCoreLocalityRatio } from './core-locality-ratio.js';
-import { estimateSingleStage,                                              } from './occupancy.js';
+import { estimateSingleStage, tailRecoveryMs, tailRemovedWorkMs, stragglerFixLongestTaskMs,                                              } from './occupancy.js';
 import { isExchangeNode, isBroadcastExchangeNode } from './plan-node-detail.js';
+import { cyrb53 } from './string-hash.js';
                                                                
 
 const MB = 1024 * 1024;
@@ -74,6 +75,9 @@ const TB = 1024 * GB;
                                            
                                          
                          
+                             
+                                 
+                               
                            
                                     
                              
@@ -154,6 +158,7 @@ const TB = 1024 * GB;
                                                                   
                                        
                                                                                            
+                        
  
 
 // SpillPressureDetector (5a) + SpillSkewDetector (5b).
@@ -206,8 +211,10 @@ export function unionStageIds(nodes            , fallback          )           {
 
 // Bottom-up shape computation for duplicate-subtree detection and the
 // cachingOpportunity composite detector. `size` is the subtree node count; the default
-// fingerprint encodes operator name + sorted metric NAMES (never values, per spec) + child
-// fingerprints, so two subtrees with the same shape but different values still collide.
+// fingerprint encodes operator name + sorted metric NAMES (never values, per spec) + a digest
+// of each child's fingerprint, so two subtrees with the same shape but different values still
+// collide. Digesting children keeps each fingerprint O(own size): embedding the full child
+// strings made every node carry its whole subtree, O(n x depth) text on deep real plans.
 //
 // opts.includeDetail (default false) folds root's own detail (via opts.normalizeDetail) into
 // ONLY root's fingerprint, never a child's: lets a caller match "this node's own detail + shape"
@@ -239,7 +246,7 @@ export function computePlanShapes(
     const childShapes = realChildren.map((c) => visit(c, false));
     const size = 1 + childShapes.reduce((sum, c) => sum + c.size, 0);
     const metricNames = (realMetrics ?? []).map((m) => m.name).sort().join(',');
-    const childFingerprints = childShapes.map((c) => c.fingerprint).join(',');
+    const childFingerprints = childShapes.map((c) => cyrb53(c.fingerprint)).join(',');
     const fingerprint = isRoot && includeDetail
       ? `${node.name}[${metricNames}]<${normalize(node.detail ?? '')}>{${childFingerprints}}`
       : `${node.name}[${metricNames}]{${childFingerprints}}`;
@@ -261,7 +268,11 @@ export function normalizeDetail(detail        )         {
     .replace(/,?\s*plan_id=\d+/g, '')
     .replace(/\[codegen id\s*:\s*\d+\]/gi, '[codegen id]')
     .replace(/\bBuild(Left|Right)\b/g, 'BuildSide');
-  s = s.replace(/([A-Za-z_][\w.]*)\s*=\s*([A-Za-z_][\w.]*)/g, (_m, l, r) => {
+  // The lookbehind only skips starts inside an identifier, which can't match unless the
+  // identifier's own start already did: same output, without re-scanning each identifier from
+  // every one of its letters (O(length^2) on the 6.8 KB average join detail of the largest real
+  // log; 64ms -> 40ms per analyze()).
+  s = s.replace(/(?<![A-Za-z_])([A-Za-z_][\w.]*)\s*=\s*([A-Za-z_][\w.]*)/g, (_m, l, r) => {
     const [a, b] = [l, r].sort();
     return `${a} = ${b}`;
   });
@@ -270,6 +281,21 @@ export function normalizeDetail(detail        )         {
 
 const JOIN_NAME_RE = /Join/i;
 
+// scanRelationId per plan node, memoized: cachingOpportunity's relation walk and
+// findCompositeCandidates both classify every node of every execution, and a JDBC scan's detail
+// carries its whole inner SQL, so the repeated regex passes were a measurable share of
+// analyze(). Keyed by node identity: a resolved plan tree is never mutated after the parser
+// posts it.
+const relationIdByNode = new WeakMap                         ();
+function relationIdOf(node          )                {
+  let rid = relationIdByNode.get(node);
+  if (rid === undefined) {
+    rid = scanRelationId(node.name ?? '', node.detail ?? '');
+    relationIdByNode.set(node, rid);
+  }
+  return rid;
+}
+
 // Structural operator kind for cachingOpportunity's composite detection: 'join' covers every
 // Spark join physical operator; 'union' is Spark's exact `Union` node. CartesianProduct is
 // deliberately excluded (out of scope, as in plan-summary.ts).
@@ -302,11 +328,12 @@ export function findCompositeCandidates(root          )                       {
     path.pop();
 
     const metricNames = (node.metrics ?? []).map((m) => m.name).sort().join(',');
-    const childFingerprints = childResults.map((r) => r.fingerprint).join(',');
+    // Child digests, as in computePlanShapes: deterministic, so still comparable across executions.
+    const childFingerprints = childResults.map((r) => cyrb53(r.fingerprint)).join(',');
     const fingerprint = `${node.name}[${metricNames}]{${childFingerprints}}`;
 
     const leafRelationBytes = new Map                ();
-    const rid = scanRelationId(node.name ?? '', node.detail ?? '');
+    const rid = relationIdOf(node);
     if (rid) {
       const bytesMetric = (node.metrics ?? []).find((m) => m.name === FILES_READ_BYTES);
       leafRelationBytes.set(rid, (leafRelationBytes.get(rid) ?? 0) + (bytesMetric ? bytesMetric.value : 0));
@@ -343,7 +370,7 @@ function firstLeafRelationId(node          )                {
   let found                = null;
   walkPlanTree(node, (n) => {
     if (found) return;
-    found = scanRelationId(n.name ?? '', n.detail ?? '');
+    found = relationIdOf(n);
   });
   return found;
 }
@@ -411,6 +438,74 @@ export function findDuplicateSubtrees(
   return results;
 }
 
+// True when every occurrence also agrees node-for-node on normalized detail (filters, columns,
+// scanned table, literals). The fingerprint ignores detail, so occurrences can be same-shaped
+// branches over different data; only identical ones are repeated work that computing once would
+// save. AQE query-stage numbers (`ShuffleQueryStage 718` vs `720`) name the same computation's
+// runtime stage and are ignored too. On the 14 real logs, 270 of 546 groups differed beyond that.
+function occurrencesHaveIdenticalDetails(occurrences            )          {
+  const detailsOf = (root          )           => {
+    const out           = [];
+    walkPlanTree(root, (n) => out.push(n.detail ?? ''));
+    return out;
+  };
+  const normalize = (d        )         => normalizeDetail(d).replace(/\b(\w+QueryStage)\s+\d+/g, '$1');
+  const first = detailsOf(occurrences[0]);
+  const firstNormalized                         = [];
+  for (const other of occurrences.slice(1)) {
+    const details = detailsOf(other);
+    if (details.length !== first.length) return false;
+    for (let i = 0; i < first.length; i++) {
+      if (details[i] === first[i]) continue;
+      firstNormalized[i] ??= normalize(first[i]);
+      if (normalize(details[i]) !== firstNormalized[i]) return false;
+    }
+  }
+  return true;
+}
+
+// Stages that run nothing but the duplicated occurrences' operators: every operator attributed to
+// the stage is inside `nodes`. A stage shared with operators outside them (the join consuming the
+// subtree, the other join side) does other work too, so its time isn't the subtree's to claim;
+// counting it also let sibling groups claim the same stage twice. A WholeStageCodegen wrapper and
+// an Exchange's write half aren't other work (the fused pipeline around an operator, the shuffle
+// write of its output), so they're left out of both counts; on the 14 real logs they were the
+// only outside node on 426 stages.
+function countsTowardStage(node          )          {
+  return node.exchangeRole !== 'write' && !node.name.startsWith('WholeStageCodegen');
+}
+
+function operatorCountByStage(nodes                    )                      {
+  const counts = new Map                ();
+  for (const node of nodes) {
+    if (!countsTowardStage(node)) continue;
+    for (const sid of node.stageIds ?? []) counts.set(sid, (counts.get(sid) ?? 0) + 1);
+  }
+  return counts;
+}
+
+function stageOperatorShares(nodes            , executionCounts                     )                         {
+  const shares                         = {};
+  for (const [sid, count] of operatorCountByStage(nodes)) shares[sid] = count / (executionCounts.get(sid) ?? count);
+  return shares;
+}
+
+// Fingerprint matching compares operator + metric names only, not literal values or expr IDs
+// (see the finding's validationRequired text), so a small pattern repeated the bare minimum
+// number of times is the case most likely to be coincidental rather than real duplicated work.
+// A bigger matched subtree, or more repeats, are each on their own strong corroborating evidence
+// that the match is real: the odds of two semantically-different query branches producing an
+// identical operator-name sequence shrink fast as the sequence grows or repeats.
+function duplicateSubtreeConfidence(
+  subtreeSize        ,
+  occurrences        ,
+  thresholds                                                    ,
+)                            {
+  if (subtreeSize <= thresholds.minSubtreeSize && occurrences <= thresholds.minOccurrences) return 'low';
+  if (subtreeSize >= thresholds.minSubtreeSize * 2 || occurrences >= thresholds.minOccurrences + 2) return 'high';
+  return 'medium';
+}
+
 // Exact metric names Spark emits, verified against a real SQLExecutionStart's sparkPlanInfo.
 // The write-side byte metric is "written output", not "size of written files".
 const FILES_READ_COUNT = 'number of files read';
@@ -505,12 +600,27 @@ function meetsRuntimeFloor(wasteMs        , appDurationMs               , floorP
   return appDurationMs == null || wasteMs >= appDurationMs * floorPct;
 }
 
+// A stage that ran for less than floorPct of the run (a known duration): an estimate clipped to
+// the stage can't reach floorPct, so every finding there grades info. A zero-length stage (no
+// submission time on older Spark) is not skipped: it gets no estimate and keeps its own band.
+function stageBelowRuntimeFloor(stage               , ctx                         , floorPct        )          {
+  const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  const appDurationMs = computeAppDurationMs(ctx);
+  return appDurationMs != null && stageDurationMs > 0 && stageDurationMs < appDurationMs * floorPct;
+}
+
 // Runs a raw waste delta through the same occupancy clip impact-estimator.ts applies before
 // display, so the runtime floor checks recoverable wall-clock, not a delta a physical floor
 // leaves unrecoverable. Falls back to the raw delta when occupancy data is unavailable.
-function clippedWasteMs(wasteMs        , stageId        , ctx              )         {
+// skew/straggler claims shorten the stage's longest task, hence shortensLongestTask (see occupancy.ts).
+function clippedWasteMs(
+  wasteMs        , stageId        , ctx                         , removedCoreWorkMs        , longestTaskAfterFixMs = 0,
+)         {
   if (!ctx) return wasteMs;
-  const est = estimateSingleStage(wasteMs, stageId, ctx.stages                                          , ctx.occupancy);
+  const est = estimateSingleStage(
+    wasteMs, stageId, ctx.stages                                          , ctx.occupancy,
+    { shortensLongestTask: true, removedCoreWorkMs, longestTaskAfterFixMs },
+  );
   return est ? est.wallClock.high : wasteMs;
 }
 
@@ -519,6 +629,19 @@ function clippedWasteMs(wasteMs        , stageId        , ctx              )
 const CACHE_UTILIZATION_VALIDATION =
   "This ratio is a point-in-time storage snapshot from stage-submission events, not a runtime read-count. Confirm against the Spark UI's Storage tab before acting.";
 
+// Both cachedRatio and diskRatio are percentages of an RDD's partition/byte count: with few
+// partitions, one partition flipping cached/evicted (or disk-resident/memory-resident) swings the
+// reported percentage by a large amount, so the point estimate is noisy. More partitions average
+// that noise out into a stable ratio. numPartitions is the only sample-size signal
+// DetectorRddInfo carries, so it drives confidence for both variants rather than a flat guess.
+// 10/50 mirror this file's other "trust the sample" cutoffs (skew's minTasksForP95: 20,
+// coreLocality's minTasks: 50).
+function cacheSampleConfidence(numPartitions        )                            {
+  if (numPartitions < 10) return 'low';
+  if (numPartitions >= 50) return 'high';
+  return 'medium';
+}
+
 function partialCacheFinding(rdd                 , cachedRatio        , impactBand                    )          {
   const rddName = rdd.name || `RDD ${rdd.id}`;
   const cachedPct = Math.round(cachedRatio * 100);
@@ -527,7 +650,7 @@ function partialCacheFinding(rdd                 , cachedRatio        , impactBa
     type: 'cacheUtilization', variant: 'partialCache', stageId: null,
     rddId: rdd.id, rddName,
     impactBand, metric: 'cachedRatio', value: cachedPct,
-    confidence: 'medium',
+    confidence: cacheSampleConfidence(rdd.numPartitions),
     validationRequired: CACHE_UTILIZATION_VALIDATION,
     memorySize: rdd.memorySize, diskSize: rdd.diskSize,
     numCachedPartitions: rdd.numCachedPartitions, numPartitions: rdd.numPartitions,
@@ -542,7 +665,7 @@ function diskSpilloverFinding(rdd                 , diskRatio        , impactBan
     type: 'cacheUtilization', variant: 'diskSpillover', stageId: null,
     rddId: rdd.id, rddName,
     impactBand, metric: 'diskRatio', value: diskPct,
-    confidence: 'medium',
+    confidence: cacheSampleConfidence(rdd.numPartitions),
     validationRequired: CACHE_UTILIZATION_VALIDATION,
     memorySize: rdd.memorySize, diskSize: rdd.diskSize,
     numCachedPartitions: rdd.numCachedPartitions, numPartitions: rdd.numPartitions,
@@ -584,6 +707,103 @@ function diskSpilloverFinding(rdd                 , diskRatio        , impactBan
 export const STRAGGLER_FLOOR_PCT_WARN = 0.005;
 export const STRAGGLER_FLOOR_PCT_CRIT = 0.02;
 
+// A ratio just past ratioWarn is the case most likely to be ordinary task-duration variance
+// rather than real skew; a ratio many multiples past it (a 50x P95/median vs. a 3.1x one) is
+// unambiguous. 1.5x/5x mirror this file's other "just past the floor vs. clearly past it" splits
+// (duplicateSubtreeConfidence's 2x, cacheSampleConfidence's 10/50-partition cutoffs).
+function skewConfidence(ratio        , ratioWarn        )                            {
+  if (ratio <= ratioWarn * 1.5) return 'low';
+  if (ratio >= ratioWarn * 5) return 'high';
+  return 'medium';
+}
+
+// warnPct100/lowInfoPct100 are this detector's own two thresholds; scale confidence as a multiple
+// of whichever one gates the branch that fired, the same way skewConfidence scales off ratioWarn.
+// High-GC: a pct just past warnPct100 (10%) is likely normal variance, 3x past it is unambiguous.
+// Low-GC ("cost", over-provisioned): a pct just under lowInfoPct100 (5%) is borderline, a pct near
+// zero is unambiguous idle GC.
+function gcConfidence(
+  pct        ,
+  thresholds                                               ,
+  direction                ,
+)                            {
+  if (direction === 'high') {
+    if (pct <= thresholds.warnPct100 * 1.5) return 'low';
+    if (pct >= thresholds.warnPct100 * 3) return 'high';
+    return 'medium';
+  }
+  if (pct >= thresholds.lowInfoPct100 * 0.66) return 'low';
+  if (pct <= thresholds.lowInfoPct100 * 0.2) return 'high';
+  return 'medium';
+}
+
+// warnFloor gates the finding, so a value just past it is the weakest evidence this detector can
+// produce. highFloor is critPct: straggler's own second (speculative-share) tier, 2x warnPct
+// (0.10 -> 0.20); reused as the high-confidence bar for the stragglerShare path too since that
+// metric has no dedicated critical tier of its own (see the "no dedicated critical tier" comment
+// on the straggler detector) but is the same 0-1 task-share magnitude.
+function stragglerConfidence(shareValue        , warnFloor        , highFloor        )                            {
+  if (shareValue < warnFloor * 1.5) return 'low';
+  if (shareValue >= highFloor) return 'high';
+  return 'medium';
+}
+
+// minWasteMs is speculationWaste's own floor; a run that barely clears it (under 1.5x) is the
+// weakest case, one that clears it several times over (4x, i.e. 4 minutes against a 1-minute
+// floor) is unambiguous.
+function speculationWasteConfidence(wastedMs        , minWasteMs        )                            {
+  if (wastedMs <= minWasteMs * 1.5) return 'low';
+  if (wastedMs >= minWasteMs * 4) return 'high';
+  return 'medium';
+}
+
+// The finding already gates on wastedMBSeconds > wasteBufferMultiplier * usedMBSeconds, i.e. a
+// ratio of 1 at the floor; scale confidence off that same ratio the way skewConfidence scales off
+// ratioWarn, instead of introducing a second, unrelated multiplier.
+function memoryWasteConfidence(wastedMBSeconds        , usedMBSeconds        , wasteBufferMultiplier        )                            {
+  const ratio = usedMBSeconds > 0 ? wastedMBSeconds / (wasteBufferMultiplier * usedMBSeconds) : Infinity;
+  if (ratio <= 1.5) return 'low';
+  if (ratio >= 3) return 'high';
+  return 'medium';
+}
+
+// Two independent weak spots can each undercut this finding: a ratio just past warnRatio (could
+// be one bad stage), or too few sampled tasks (mirrors cacheSampleConfidence's use of
+// numPartitions as a sample-size signal). Report whichever signal is weaker rather than
+// averaging them away. critRatio is this detector's own existing second tier, reused directly as
+// the ratio high-bar; minTasks*2/*4 mirror the same "2x floor is still weak, 4x is strong" spread
+// used elsewhere in this file.
+function coreLocalityConfidence(
+  ratio        ,
+  totalTasks        ,
+  thresholds                                                            ,
+)                            {
+  const rank = { low: 0, medium: 1, high: 2 }         ;
+  const ratioTier = ratio < thresholds.warnRatio * 1.5 ? 'low' : ratio >= thresholds.critRatio ? 'high' : 'medium';
+  const sampleTier = totalTasks < thresholds.minTasks * 2 ? 'low' : totalTasks >= thresholds.minTasks * 4 ? 'high' : 'medium';
+  return rank[ratioTier] <= rank[sampleTier] ? ratioTier : sampleTier;
+}
+
+// warningPct/criticalPct are this detector's own two tiers (0.30/0.60); a churn rate just past
+// warningPct is the borderline call the impact-band split already treats as the weaker tier, so
+// reuse criticalPct directly as the high-confidence bar instead of inventing a third figure.
+function autoscalingChurnConfidence(shortLivedPct        , warningPct        , criticalPct        )                            {
+  if (shortLivedPct <= warningPct * 1.5) return 'low';
+  if (shortLivedPct >= criticalPct) return 'high';
+  return 'medium';
+}
+
+// minExecutions is the bare minimum occurrence count this detector will even emit a finding for;
+// a match at exactly that count is the weakest reuse signal (as likely to be coincidental overlap
+// as real shared work), while 3x the floor is several independent executions all hitting the same
+// relation/composite shape, unambiguous. Mirrors duplicateSubtreeConfidence's occurrences handling
+// for the same reason: repetition count is the strength signal for a structural-match detector.
+function cachingReuseConfidence(occurrences        , minExecutions        )                            {
+  if (occurrences <= minExecutions) return 'low';
+  if (occurrences >= minExecutions * 3) return 'high';
+  return 'medium';
+}
+
 export const DETECTORS             = [
   {
     type: 'skew', scope: 'stage', order: 30, fixEffort: 'code', version: 1,
@@ -602,17 +822,18 @@ export const DETECTORS             = [
       if (ratio <= this.thresholds.ratioWarn) return null;
       // Same absolute delta impact-estimator.ts's 'skew' case reports as savings; clipped the
       // same way before the floor check so the gate agrees with what's displayed.
-      const wasteMs = Math.max(0, metric === 'P95/median' ? stage.taskDurationP95 - stage.taskDurationP50 : stage.taskDurationMax - stage.taskDurationP50);
+      const singleDelta = Math.max(0, metric === 'P95/median' ? stage.taskDurationP95 - stage.taskDurationP50 : stage.taskDurationMax - stage.taskDurationP50);
+      const wasteMs = tailRecoveryMs(stage, singleDelta);
       const appDurationMs = computeAppDurationMs(ctx);
-      const floorWasteMs = clippedWasteMs(wasteMs, stage.id, ctx);
+      const floorWasteMs = clippedWasteMs(wasteMs, stage.id, ctx, tailRemovedWorkMs(stage, singleDelta), stragglerFixLongestTaskMs(stage));
       if (!meetsRuntimeFloor(floorWasteMs, appDurationMs, this.thresholds.floorPctWarn)) return null;
       const value = Math.round(ratio * 10) / 10;
       return {
         type: 'skew', stageId: stage.id,
         impactBand: 'warning',
         metric, value,
-        confidence: 'low',
-        validationRequired: 'The 0.5% runtime-floor percentage that gates this finding is our own noise floor, unvalidated: no external tool publishes an equivalent metric to calibrate against. Confirm against known-good/known-bad real logs before trusting the impact-band split.',
+        confidence: skewConfidence(ratio, this.thresholds.ratioWarn),
+        validationRequired: 'This finding is gated by a 0.5% runtime-floor threshold, our own noise floor for this metric.',
         recommendation: `Task duration ratio (${metric}) is ${value}×: for join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key to reduce task skew.`,
       };
     },
@@ -620,9 +841,13 @@ export const DETECTORS             = [
   {
     type: 'stageShape', scope: 'stage', order: 35, fixEffort: 'code', version: 1,
     docAnchor: '#bottleneck-stage-shape',
-    thresholds: { pRatioMax: 0.5, oiRatioMax: 10, skewWarn: 3 },
+    // lowParallelismFloorPct: the same 0.5% runtime floor the tiered detectors use. Parallelizing a
+    // stage can't save more than the stage's own duration, so a shorter stage can't clear it; on
+    // the 14 real logs that was 2839 of 3005 lowParallelism findings (2168 on sub-second stages).
+    // App-wide idle capacity stays covered by utilization.
+    thresholds: { pRatioMax: 0.5, oiRatioMax: 10, skewWarn: 3, lowParallelismFloorPct: 0.005 },
     detect(
-                                                                                        
+                                                                                                                        
       stage               ,
       ctx              ,
     )            {
@@ -630,8 +855,9 @@ export const DETECTORS             = [
       const execCount = (stage.executorStats ?? []).length;
       const cores = ctx?.app?.resources?.executor?.cores ?? 1;
       const totalCores = execCount * cores;
+      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
       // PRatio: under-parallelization.
-      if (totalCores > 0) {
+      if (totalCores > 0 && !stageBelowRuntimeFloor(stage, ctx, this.thresholds.lowParallelismFloorPct)) {
         const pRatio = stage.taskCount / totalCores;
         if (pRatio < this.thresholds.pRatioMax) {
           out.push({
@@ -657,7 +883,6 @@ export const DETECTORS             = [
       // TaskStageSkew: straggler cost vs stage wall-clock. Skip near-zero duration. Always info
       // like its siblings: this trigger forces the occupancy-clipped estimate to exactly zero on
       // every firing, so there's no wall-clock-backed tier left to gate on.
-      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
       if (stageDurationMs > 0) {
         const ratio = stage.taskDurationMax / stageDurationMs;
         if (ratio > this.thresholds.skewWarn) {
@@ -676,13 +901,18 @@ export const DETECTORS             = [
   {
     type: 'shuffle', scope: 'stage', order: 20, fixEffort: 'config', version: 1,
     docAnchor: '#bottleneck-shuffle',
-    thresholds: { minBytes: 50 * MB },
+    // stageFloorPct: the 0.5% runtime floor. The shuffle claim is clipped to the stage, so on a
+    // shorter stage it graded info: 182 of 284 shuffle findings on the 14 real logs. The shuffle
+    // is still there on those stages; the floor is why they're dropped.
+    thresholds: { minBytes: 50 * MB, stageFloorPct: 0.005 },
     detect(
-                                                 
+                                                                        
       stage               ,
+      ctx              ,
     )                 {
       const bytes = stage.shuffleReadBytes;
       if (bytes <= this.thresholds.minBytes) return null;
+      if (stageBelowRuntimeFloor(stage, ctx, this.thresholds.stageFloorPct)) return null;
       return {
         type: 'shuffle', stageId: stage.id,
         impactBand: 'info',
@@ -693,7 +923,7 @@ export const DETECTORS             = [
   },
   {
     type: 'partitionSizing', scope: 'stage', order: 22, fixEffort: 'config', version: 1,
-    docAnchor: '#bottleneck-shuffle',
+    docAnchor: '#bottleneck-partition-sizing',
     thresholds: { skewRatio: 5, skewFloorBytes: 256 * MB, lowParTotalBytes: GB, lowParMaxTasks: 7, maxPartBytes: 5 * GB },
     detect(
              
@@ -742,9 +972,12 @@ export const DETECTORS             = [
   {
     type: 'spill', scope: 'stage', order: 10, fixEffort: 'code', version: 1,
     docAnchor: '#bottleneck-spill',
-    thresholds: { singleTaskDiskGiB: 1, singleTaskMemGiB: 4, highDiskGiB: 1, highTaskDiskMB: 512, highMemGiB: 4, medDiskMB: 256, medMemGiB: 1, skewRatio: 5, skewDiskFloorMB: 128, skewMemFloorMB: 256, skewMinTasks: 10 },
-    detect(                                       stage               )                 {
+    // stageFloorPct: the 0.5% runtime floor, as for shuffle (12 of 38 spill findings on the 14 real
+    // logs, all info). The spill is still there on those stages; the floor is why they're dropped.
+    thresholds: { singleTaskDiskGiB: 1, singleTaskMemGiB: 4, highDiskGiB: 1, highTaskDiskMB: 512, highMemGiB: 4, medDiskMB: 256, medMemGiB: 1, skewRatio: 5, skewDiskFloorMB: 128, skewMemFloorMB: 256, skewMinTasks: 10, stageFloorPct: 0.005 },
+    detect(                                       stage               , ctx              )                 {
       if (stage.memoryBytesSpilled === 0) return null;
+      if (stageBelowRuntimeFloor(stage, ctx, this.thresholds.stageFloorPct)) return null;
       const cls = stage.spillClassification;
       const classified = cls === 'skew' || cls === 'volume';
       const mag = computeSpillMagnitude(stage, this.thresholds);
@@ -766,21 +999,26 @@ export const DETECTORS             = [
   {
     type: 'gc', scope: 'stage', order: 50, fixEffort: 'config', version: 1,
     docAnchor: '#bottleneck-gc',
-    confidence: 'low',
-    validationRequired: 'The 10-second minimum-runtime floor that gates this finding is our own noise floor, unvalidated: no external tool publishes an equivalent metric to calibrate against. Confirm against known-good/known-bad real logs before trusting the impact-band split.',
+    validationRequired: 'This finding is gated by a 10-second minimum-runtime floor, our own noise floor for this metric.',
     thresholds: {
       warnPct100: 10,
       // Descending tier: ExecutorGcHeuristic, ported as-is.
       lowInfoPct100: 5,
       // NOT SOURCED: our own noise floor so a stage that barely ran doesn't flag either direction.
       minRunTimeMs: 10000,
+      // lowInfoFloorPct: the 0.5% runtime floor stageShape's lowParallelism uses. The low-GC note is
+      // an app-level memory-sizing signal; on a stage shorter than this share of the run it adds
+      // nothing to that call. On the 14 real logs that was 464 of 685 low-GC notes (of 701 gc
+      // findings); the low-GC pattern is still true on those stages, the floor is why they're dropped.
+      lowInfoFloorPct: 0.005,
     },
     detect(
              
-                                                                                        
-                                                       
+                                                                                                                 
+                                   
         
       stage               ,
+      ctx              ,
     )                 {
       const pct = stage.gcPct;
       if ((stage.executorRunTime ?? 0) >= this.thresholds.minRunTimeMs
@@ -790,19 +1028,20 @@ export const DETECTORS             = [
           type: 'gc', stageId: stage.id,
           impactBand: 'warning',
           metric: 'gcPct', value,
-          confidence: this.confidence, validationRequired: this.validationRequired,
+          confidence: gcConfidence(pct, this.thresholds, 'high'), validationRequired: this.validationRequired,
           recommendation: `GC consumed ${value}% of executor run time: reduce object creation, use primitive types, avoid UDFs, increase executor memory.`,
         };
       }
       // Low-GC (cost) branch: only for stages that ran long enough to be meaningful.
       if ((stage.executorRunTime ?? 0) >= this.thresholds.minRunTimeMs
-          && pct < this.thresholds.lowInfoPct100) {
+          && pct < this.thresholds.lowInfoPct100
+          && !stageBelowRuntimeFloor(stage, ctx, this.thresholds.lowInfoFloorPct)) {
         const value = Math.round(pct * 10) / 10;
         return {
           type: 'gc', stageId: stage.id, direction: 'low',
           impactBand: 'info',
           metric: 'gcPct', value,
-          confidence: this.confidence, validationRequired: this.validationRequired,
+          confidence: gcConfidence(pct, this.thresholds, 'low'), validationRequired: this.validationRequired,
           recommendation: `GC consumed only ${value}% of executor run time: memory may be over-provisioned; consider reducing spark.executor.memory for cost savings.`,
         };
       }
@@ -818,19 +1057,29 @@ export const DETECTORS             = [
       // stages, sub-second/sub-64MB host differences produce huge noise ratios. 1s is above
       // per-task jitter but below genuine slow-host stages; 64MB mirrors the spill disk-skew floor.
       floorMs: 1000, floorBytes: 64 * MB,
+      // stageFloorPct: the tiered detectors' 0.5% runtime floor. On a stage shorter than this share
+      // of the run a slow host can't cost that much: duration findings are clipped to the stage
+      // and graded info, and a byte-dimension imbalance (no time estimate, so its ratio tier was
+      // its band) was graded info here since 6298149. So the stage is skipped: on the 14 real logs
+      // 324 of 452 slowHost findings, all info. The imbalance is still there on those stages; the
+      // floor is why they're dropped.
+      stageFloorPct: 0.005,
     },
     detect(
              
                      
                                                                                   
                                                                                                               
+                                
           
         
       stage               ,
+      ctx              ,
     )                   {
       const hosts = stage.hostStats ?? [];
       const execs0 = stage.executorStats ?? [];
       if ((hosts.length < this.thresholds.minHosts && execs0.length < this.thresholds.minHosts) || stage.taskCount < this.thresholds.minTasks) return null;
+      if (stageBelowRuntimeFloor(stage, ctx, this.thresholds.stageFloorPct)) return null;
       const out            = [];
       if (hosts.length >= this.thresholds.minHosts) {
         const means = hosts.map(h => ({ host: h.host, taskCount: h.taskCount, mean: h.totalDuration / h.taskCount }));
@@ -985,31 +1234,51 @@ export const DETECTORS             = [
     docAnchor: '#bottleneck-straggler',
     // floorPctWarn/floorPctCrit are re-exported as STRAGGLER_FLOOR_PCT_WARN/CRIT and reused as
     // impact-band.ts's global noise floor: keep the two in sync.
-    thresholds: { minTasks: 10, shareWarn: 0.05, warnPct: 0.10, critPct: 0.20, floorPctWarn: STRAGGLER_FLOOR_PCT_WARN, floorPctCrit: STRAGGLER_FLOOR_PCT_CRIT },
+    // shareWarnAtFloor: in a large stage, the few stragglers that gate it for tens of seconds can be
+    // only 2.5-5% of its tasks. Scored against a task-level replay of every stage on 14 real logs
+    // (recoverable = replay with each task over 4x P50 capped at P50), admitting 2.5-5% shares
+    // whose clipped tail already clears floorPctWarn found 3 such stages (10-48s) for 1 borderline
+    // miss; admitting every 2.5% share instead added 86 findings below the floor.
+    // A stage shorter than floorPctWarn of the run is skipped outright: its tail can't cost more
+    // than the stage's own duration, so every finding there graded info. On the 14 real logs that
+    // was 671 of 753 straggler findings, none above info; the slow tail is still real on those
+    // stages, the floor is why they're dropped.
+    thresholds: { minTasks: 10, shareWarn: 0.05, shareWarnAtFloor: 0.025, warnPct: 0.10, critPct: 0.20, floorPctWarn: STRAGGLER_FLOOR_PCT_WARN, floorPctCrit: STRAGGLER_FLOOR_PCT_CRIT },
     detect(
              
-                                                                                                                                          
+                     
+                                                                                                          
+                                                     
+          
         
       stage               ,
       ctx              ,
     )                 {
       if (stage.taskCount < this.thresholds.minTasks) return null;
+      const appDurationMs = computeAppDurationMs(ctx);
+      if (stageBelowRuntimeFloor(stage, ctx, this.thresholds.floorPctWarn)) return null;
       const stragglerShare = (stage.stragglerCount ?? 0) / stage.taskCount;
-      if ((stage.speculativeTasks ?? 0) === 0 && stragglerShare <= this.thresholds.shareWarn) return null;
       const useSpeculative = (stage.speculativeTasks ?? 0) > 0;
+      if (!useSpeculative && stragglerShare <= this.thresholds.shareWarnAtFloor) return null;
       const speculativeShare = useSpeculative ? stage.speculativeTasks / stage.taskCount : 0;
       // Same absolute delta impact-estimator.ts's straggler/stageShape case reports as savings: a
       // high straggler/speculative share on a stage whose tasks barely vary models near-zero
       // savings, so it must not outrank 'info'. Clipped the same way before the floor check.
-      const wasteMs = Math.max(0, stage.taskDurationMax - stage.taskDurationP50);
-      const appDurationMs = computeAppDurationMs(ctx);
-      const floorWasteMs = clippedWasteMs(wasteMs, stage.id, ctx);
+      const longestTaskAfterFixMs = stragglerFixLongestTaskMs(stage);
+      const singleDelta = Math.max(0, stage.taskDurationMax - longestTaskAfterFixMs);
+      const wasteMs = tailRecoveryMs(stage, singleDelta);
+      const floorWasteMs = clippedWasteMs(wasteMs, stage.id, ctx, tailRemovedWorkMs(stage, singleDelta), longestTaskAfterFixMs);
       const meetsWarnFloor = meetsRuntimeFloor(floorWasteMs, appDurationMs, this.thresholds.floorPctWarn);
       const meetsCritFloor = meetsRuntimeFloor(floorWasteMs, appDurationMs, this.thresholds.floorPctCrit);
+      // The lower gate needs positive evidence the tail matters: meetsRuntimeFloor passes by
+      // default when the app's duration is unknown (an incomplete run), which isn't that.
+      const stragglerShareFires = stragglerShare > this.thresholds.shareWarn
+        || (stragglerShare > this.thresholds.shareWarnAtFloor && appDurationMs != null && meetsWarnFloor);
+      if (!useSpeculative && !stragglerShareFires) return null;
       const speculativeTier = speculativeShare >= this.thresholds.critPct && meetsCritFloor ? 'critical'
                              : speculativeShare >= this.thresholds.warnPct && meetsWarnFloor ? 'warning' : 'info';
       // Straggler share has no dedicated critical tier per detector-contract.md; only warning.
-      const stragglerTier = stragglerShare > this.thresholds.shareWarn && meetsWarnFloor ? 'warning' : 'info';
+      const stragglerTier = stragglerShareFires && meetsWarnFloor ? 'warning' : 'info';
       // Fixed fallback: overwritten by deriveImpactBand when this finding gets a real wallClock
       // estimate (the common case). Only surfaces on the rare occupancy-sweep miss.
       const impactBand = 'info';
@@ -1028,20 +1297,21 @@ export const DETECTORS             = [
         unit: useSpeculativeMetric ? 'count' : 'pct',
         speculativeTasks: stage.speculativeTasks ?? 0,
         stragglerCount: stage.stragglerCount ?? 0,
-        confidence: 'low',
-        validationRequired: 'The 0.5%/2% runtime-floor percentages that gate this finding are our own noise floor, unvalidated: no external tool publishes an equivalent metric to calibrate against. Confirm against known-good/known-bad real logs before trusting the impact-band split.',
+        confidence: useSpeculativeMetric
+          ? stragglerConfidence(speculativeShare, this.thresholds.warnPct, this.thresholds.critPct)
+          : stragglerConfidence(stragglerShare, this.thresholds.shareWarn, this.thresholds.critPct),
+        validationRequired: 'This finding is gated by 0.5%/2% runtime-floor thresholds, our own noise floor for this metric.',
         recommendation: `${detail}: rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that's a candidate for AQE's skew-join handling.`,
       };
     },
   },
   {
     type: 'speculationWaste', scope: 'stage', order: 71, fixEffort: 'config', version: 1,
-    docAnchor: '#bottleneck-straggler', confidence: 'low',
+    docAnchor: '#bottleneck-speculation-waste',
     thresholds: { minWasted: 5, minWasteMs: 60000 },
     detect(
              
                                                               
-                           
         
       stage               ,
     )                 {
@@ -1052,7 +1322,7 @@ export const DETECTORS             = [
         type: 'speculationWaste', stageId: stage.id,
         impactBand: 'warning',
         metric: 'speculationWasteMs', value: wastedMs,
-        confidence: this.confidence,
+        confidence: speculationWasteConfidence(wastedMs, this.thresholds.minWasteMs),
         recommendation: `Speculative execution discarded ${Math.round(wastedMs / 1000)}s of executor time in this stage; if task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.`,
       };
     },
@@ -1083,12 +1353,17 @@ export const DETECTORS             = [
   {
     type: 'tinyTask', scope: 'stage', order: 80, fixEffort: 'code', version: 1,
     docAnchor: '#bottleneck-tiny-tasks',
-    thresholds: { minTasks: 100, maxP50: 500, maxP95: 1000 },
+    // stageFloorPct: the tiered detectors' 0.5% runtime floor. Coalescing can't save more than the
+    // stage's own duration, so on a shorter stage every finding graded info: on the 14 real logs
+    // 132 of 151 tinyTask findings. The tasks are still tiny there; the floor is why they're dropped.
+    thresholds: { minTasks: 100, maxP50: 500, maxP95: 1000, stageFloorPct: 0.005 },
     detect(
-                                                                                 
+                                                                                                        
       stage               ,
+      ctx              ,
     )                 {
       if (stage.taskCount < this.thresholds.minTasks) return null;
+      if (stageBelowRuntimeFloor(stage, ctx, this.thresholds.stageFloorPct)) return null;
       if (stage.taskDurationP50 > this.thresholds.maxP50 || stage.taskDurationP95 > this.thresholds.maxP95) return null;
       const coalesceTo = Math.max(1, Math.round(stage.taskCount / 10));
       const fix = stage.shuffleReadBytes > 0
@@ -1124,23 +1399,42 @@ export const DETECTORS             = [
                                                    
       ctx             ,
     )                 {
-      const { app, stages } = ctx;
+      const { app, stages, executorsAdded, executorsRemoved } = ctx;
       // Nullish (not falsy) check: a literal startTime:0 must not be treated as "missing".
       if (!app || app.startTime == null || stages.size === 0) return null;
-      let firstTaskLaunch = Infinity;
+      let firstStageSubmitted = Infinity;
       for (const stage of stages.values()) {
-        if (stage.submittedAt > 0 && stage.submittedAt < firstTaskLaunch) firstTaskLaunch = stage.submittedAt;
+        if (stage.submittedAt > 0 && stage.submittedAt < firstStageSubmitted) firstStageSubmitted = stage.submittedAt;
       }
       // No stage ever recorded a submission timestamp: no basis to measure a startup gap against.
-      // Exposed now that a literal app.startTime:0 no longer short-circuits this detector entirely.
-      if (!Number.isFinite(firstTaskLaunch)) return null;
-      const gapSeconds = (firstTaskLaunch - app.startTime) / 1000;
+      if (!Number.isFinite(firstStageSubmitted)) return null;
+      // The wait is from the first runnable stage to the first executor, not from app start: the
+      // driver's own startup before its first job (36-47s on every real log, whatever the
+      // executors did) isn't something executors could shorten. On the 9 real logs that fired,
+      // the first executor arrived 146-192s after the first stage on two whose old gap read ~40s,
+      // and 2s after it on one the old gap flagged critical at 42s.
+      // An executor added before the first stage only counts if it was still alive at submission:
+      // with dynamic allocation scaling to zero, early executors can idle out before any job runs.
+      const removedAt = new Map                ();
+      for (const ev of executorsRemoved) removedAt.set(ev.executorId, ev.timestamp);
+      let firstExecutorAdded = Infinity;
+      for (const e of executorsAdded) {
+        if (!(e.timestamp > 0)) continue;
+        if (e.timestamp <= firstStageSubmitted) {
+          const removed = removedAt.get(e.executorId);
+          if (removed == null || removed > firstStageSubmitted) return null;
+        } else if (e.timestamp < firstExecutorAdded) {
+          firstExecutorAdded = e.timestamp;
+        }
+      }
+      if (!Number.isFinite(firstExecutorAdded)) return null;
+      const gapSeconds = (firstExecutorAdded - firstStageSubmitted) / 1000;
       if (gapSeconds <= this.thresholds.gapSeconds) return null;
       const value = Math.round(gapSeconds);
       return {
         type: 'coldStart', stageId: null, impactBand: 'warning',
         metric: 'startupGapSeconds', value,
-        recommendation: `The first task waited ${value}s for executors to become available: keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it doesn't scale up from zero.`,
+        recommendation: `The first stage waited ${value}s for an executor to become available: keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it doesn't scale up from zero.`,
       };
     },
   },
@@ -1298,8 +1592,8 @@ export const DETECTORS             = [
           out.push({
             type: 'memoryUtilization', variant: 'wasteModel', stageId: null,
             impactBand: 'info', metric: 'wastedMBSeconds', value,
-            confidence: 'low',
-            validationRequired: 'Memory-waste estimate uses allocated-vs-used memory-time and an unverified 1.5x buffer: confirm against the Spark UI before acting.',
+            confidence: memoryWasteConfidence(wastedMBSeconds, usedMBSeconds, this.thresholds.wasteBufferMultiplier),
+            validationRequired: 'Memory-waste estimate uses allocated-vs-used memory-time and a 1.5x buffer: confirm against the Spark UI before acting.',
             recommendation: `Allocated executor memory sat largely idle over the run (~${value.toLocaleString('en-US')} MB-seconds wasted): review spark.executor.memory and executor count.`,
           });
         }
@@ -1313,7 +1607,7 @@ export const DETECTORS             = [
     // block-access events, so a literal cache hit rate isn't derivable). Two per-RDD tiered
     // checks over rddInfo snapshots: partial caching and disk spillover. An RDD can produce both.
     type: 'cacheUtilization', scope: 'app', order: 103, fixEffort: 'code', version: 1,
-    docAnchor: '#memory-model',
+    docAnchor: '#bottleneck-cache-utilization',
     thresholds: {
       cachedRatioWarn: 0.50, cachedRatioInfo: 0.90,
       diskRatioWarn: 0.40, diskRatioInfo: 0.15,
@@ -1357,7 +1651,7 @@ export const DETECTORS             = [
     // half of the "Wasted Cores Ratio" (idle-core half is memoryUtilization's idleCores).
     // NO_PREF stays in the denominator only: shuffle-read stages legitimately report it.
     type: 'coreLocality', scope: 'app', order: 103, fixEffort: 'config', version: 1,
-    docAnchor: '#bottleneck-utilization',
+    docAnchor: '#bottleneck-core-locality',
     thresholds: { minTasks: 50, warnRatio: 0.15, critRatio: 0.35 },
     detect(
                                                                                        
@@ -1376,8 +1670,8 @@ export const DETECTORS             = [
         metric: 'nonLocalRatio', value,
         // Raw count behind the ratio, for the impact estimator. Non-null whenever totalTasks is.
         nonLocalTaskCount: nonLocalTasks ,
-        confidence: 'low',
-        validationRequired: 'The 15%/35% non-local-ratio thresholds (and the 50-task minimum) are unvalidated design-spike values: no external tool publishes an equivalent metric to calibrate against. Confirm against known-good/known-bad real logs before trusting the impact-band split.',
+        confidence: coreLocalityConfidence(ratio , totalTasks, this.thresholds),
+        validationRequired: 'This finding is gated by 15%/35% non-local-ratio thresholds (and a 50-task minimum), our own noise floor for this metric.',
         recommendation: `${value}% of tasks (${nonLocalTasks }) ran without process- or node-local data placement: check spark.locality.wait settings and executor/data colocation.`,
       };
     },
@@ -1387,12 +1681,11 @@ export const DETECTORS             = [
     // re-provisioning, not normal scale-down). Reuses utilization's add/remove matching, but
     // measures lifetime against a threshold instead of aggregate active-time.
     type: 'autoscalingChurn', scope: 'app', order: 103, fixEffort: 'config', version: 1,
-    confidence: 'low',
+    docAnchor: '#bottleneck-autoscaling-churn',
     thresholds: { shortLivedMs: 120_000, warningPct: 0.30, criticalPct: 0.60, minExecutors: 5 },
     detect(
              
                                                                                                             
-                           
         
       ctx             ,
     )                 {
@@ -1421,7 +1714,7 @@ export const DETECTORS             = [
         metric: 'shortLivedExecutorPct', value: pct,
         // Raw count behind the percentage, for the impact estimator's startup-overhead figure.
         shortLivedExecutorCount: shortLivedCount,
-        confidence: this.confidence,
+        confidence: autoscalingChurnConfidence(shortLivedPct, this.thresholds.warningPct, this.thresholds.criticalPct),
         recommendation: `${pct}% of executors ran for under 2 minutes before being removed. This looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
       };
     },
@@ -1430,7 +1723,7 @@ export const DETECTORS             = [
     // Cross-execution relation reuse: flags an input relation scanned by two or more SQL
     // executions in one run, firing on real relation names (parquet:..., jdbc:...).
     type: 'cachingOpportunity', scope: 'app', order: 105, fixEffort: 'code', version: 1,
-    docAnchor: '#bottleneck-utilization',
+    docAnchor: '#bottleneck-caching-opportunity',
     thresholds: { minExecutions: 2 },
     detect(                                                 ctx             )                   {
       const sql = ctx.sql;
@@ -1457,7 +1750,7 @@ export const DETECTORS             = [
         // Dedupe relations within one execution (self-joins count once), summing read bytes per relation.
         const perExec = new Map                ();
         walkPlanTree(exec.planTree, (node) => {
-          const rid = scanRelationId(node.name ?? '', node.detail ?? '');
+          const rid = relationIdOf(node);
           if (!rid) return;
           const bytesMetric = (node.metrics ?? []).find(m => m.name === FILES_READ_BYTES);
           perExec.set(rid, (perExec.get(rid) ?? 0) + (bytesMetric ? bytesMetric.value : 0));
@@ -1562,7 +1855,7 @@ export const DETECTORS             = [
           metric: 'executionReuse', value,
           format: 'derived', relations, operator: agg.operator, relation: relationDisplay,
           executionIds: finalExecutionIds, totalReadBytes,
-          confidence: 'low',
+          confidence: cachingReuseConfidence(value, this.thresholds.minExecutions),
           validationRequired:
             'Composite reuse is inferred from a structural plan-shape match (operator + normalized ' +
             'join/filter condition + child shapes) across SQL executions; confirm these executions ' +
@@ -1593,7 +1886,7 @@ export const DETECTORS             = [
           relation: agg.relation, format: agg.format,
           executionIds: residualExecutionIds.sort((a, b) => a - b),
           totalReadBytes,
-          confidence: 'low',
+          confidence: cachingReuseConfidence(value, this.thresholds.minExecutions),
           validationRequired:
             'Relation-reuse is inferred from the pre-AQE plan scan identity across SQL ' +
             'executions; confirm the reads are the same data and cacheable within one ' +
@@ -1721,9 +2014,13 @@ export const DETECTORS             = [
   {
     type: 'duplicatePlanSubtree', scope: 'sql', order: 130, fixEffort: 'code', version: 2,
     docAnchor: '#bottleneck-duplicate-plan-subtree',
-    thresholds: { minSubtreeSize: 3, minOccurrences: 2 },
+    // stageFloorPct: the 0.5% runtime floor. The claim counts at most each linked stage's own
+    // task-active time, so a repeat whose stages together lasted less than this share of the run
+    // graded info: 340 of 545 findings on the 14 real logs. The repeat is still in the plan; the
+    // floor is why they're dropped. A repeat with no linked stage time is kept.
+    thresholds: { minSubtreeSize: 3, minOccurrences: 2, stageFloorPct: 0.005 },
     detect(
-                                                                               
+                                                                                                      
       sqlExec                 ,
       ctx             ,
     )                   {
@@ -1731,26 +2028,44 @@ export const DETECTORS             = [
       const groups = findDuplicateSubtrees(sqlExec.planTree, this.thresholds);
       if (groups.length === 0) return null;
       const fallbackStageIds = stageIdsForSqlExec(sqlExec.id, ctx.stages);
-      return groups.map((g) => {
+      const executionNodes             = [];
+      walkPlanTree(sqlExec.planTree, (node) => executionNodes.push(node));
+      const operatorsByStage = operatorCountByStage(executionNodes);
+      const appDurationMs = computeAppDurationMs(ctx);
+      const findings = groups.map((g)                 => {
         const nodes             = [];
         for (const n of g.nodes) walkPlanTree(n, (node) => nodes.push(node));
         const stageIds = unionStageIds(nodes, fallbackStageIds);
+        let stagesMs = 0;
+        for (const id of stageIds) {
+          const stage = ctx.stages.get(id);
+          if (stage) stagesMs += Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
+        }
+        if (appDurationMs != null && stagesMs > 0 && stagesMs < appDurationMs * this.thresholds.stageFloorPct) return null;
+        const occurrencesIdentical = occurrencesHaveIdenticalDetails(g.nodes);
+        const stageShares = stageOperatorShares(nodes, operatorsByStage);
         // resolvePlanTree always sets id; safe downstream of it.
         const planNodeIds = nodes.map((n) => n.id ).filter(Boolean);
         const touching = g.sampleRelation ? ` (touching ${g.sampleRelation})` : '';
+        const differing = occurrencesIdentical ? '' : ' Their filters, columns or scanned tables differ, so the repeats may compute different data.';
         return {
           type: 'duplicatePlanSubtree', executionId: sqlExec.id, stageIds, planNodeIds,
+          stageShares, occurrencesIdentical,
           // Fixed fallback: overwritten by deriveImpactBand when this finding gets a real
-          // wallClock estimate (the common case). Only surfaces on the rare occupancy-sweep miss.
-          impactBand: 'warning', metric: 'subtreeOccurrences', value: g.occurrences,
+          // wallClock estimate. Repeats with differing details get no estimate (nothing is
+          // known to be recomputed), and neither does a subtree with no stage of its own.
+          impactBand: occurrencesIdentical && Object.keys(stageShares).length > 0 ? 'warning' : 'info',
+          metric: 'subtreeOccurrences', value: g.occurrences,
           rootName: g.rootName, subtreeSize: g.subtreeSize, sampleRelation: g.sampleRelation,
-          groupIndex: g.groupIndex, confidence: 'medium',
+          groupIndex: g.groupIndex,
+          confidence: occurrencesIdentical ? duplicateSubtreeConfidence(g.subtreeSize, g.occurrences, this.thresholds) : 'low',
           validationRequired: 'Duplicate-subtree matching compares operator names and metric names only, not literal values or expr IDs: confirm the repeated work is real in the Spark SQL plan tab before acting.',
-          recommendation: g.isExchangeRoot
+          recommendation: (g.isExchangeRoot
             ? `A ${g.subtreeSize}-node subtree rooted at ${pathBasename(g.rootName)} repeats ${g.occurrences}x in this plan${touching}: this looks like a possible missed exchange reuse; check whether the same shuffle could be computed once and reused.`
-            : `A ${g.subtreeSize}-node subtree rooted at ${pathBasename(g.rootName)} repeats ${g.occurrences}x in this plan${touching}: consider caching/persisting the shared computation or check for a duplicated query branch.`,
+            : `A ${g.subtreeSize}-node subtree rooted at ${pathBasename(g.rootName)} repeats ${g.occurrences}x in this plan${touching}: consider caching/persisting the shared computation or check for a duplicated query branch.`) + differing,
         };
-      });
+      }).filter((f)               => f !== null);
+      return findings.length > 0 ? findings : null;
     },
   },
   {