sparkforensics-cli 0.1.0

package/vendor-core/detectors.js

@@ -0,0 +1,1989 @@
+import { pathBasename, formatBytes, nsToMs, IMPACT_BAND_ORDER } from './format-utils.js';
+import { scanRelationId } from './plan-summary.js';
+import { computeTotalCores } from './core-count.js';
+import { walkPlanTree } from './plan-tree-walk.js';
+import { computeCoreLocalityRatio } from './core-locality-ratio.js';
+import { estimateSingleStage,                                              } from './occupancy.js';
+                                                               
+
+const MB = 1024 * 1024;
+const GB = 1024 * MB;
+const TB = 1024 * GB;
+
+// ---------------------------------------------------------------------------
+// Local runtime shapes.
+//
+// `types.ts`'s `Stage`/`SqlExecution`/`SparkAppInfo`/`AppModel` describe the
+// *posted* AppModel surface at a level of detail that suits the view layer
+// (both carry a catch-all `[key: string]: unknown` index signature for
+// anything beyond their few named fields). This file needs the FULL set of
+// fields `finalizeStage` (src/stage-quantiles.ts) actually computes and
+// `event-handlers.ts`'s stage/sql/app records actually carry, accessed
+// directly (arithmetic, comparisons) rather than read-and-display, so an
+// index-signature-shaped type would force an `unknown` cast at nearly every
+// field access. Every field below is verified against a real access in this
+// file (or a helper it calls); none are speculative.
+//
+// `analyzer.js` (the only real caller of `detect()`, still plain JS) passes
+// whatever the parser actually produced, so these types describe reality,
+// not a narrowing of some existing stricter type; there is nothing unsound
+// about them being independent of `types.ts`'s `Stage`/`SqlExecution`.
+                                                                                      
+                                
+                                                               
+                                                                          
+ 
+                                                                  
+                                                                   
+// Per-executor snapshot from a StageExecutorMetrics event (mergeStageRddInfo/
+// onStageExecutorMetrics): a loose bag of Spark's ExecutorMetrics field
+// names, only a few of which any detector reads.
+                                           
+                         
+                               
+                                
+                                    
+ 
+
+                                
+             
+                                 
+                    
+                      
+                     
+                      
+                      
+                      
+                           
+                         
+                         
+                             
+                                                          
+                        
+                       
+                        
+                       
+                
+                          
+                          
+                          
+                          
+                          
+                                 
+                                         
+                                                                
+                                           
+                                         
+                         
+                           
+                                    
+                             
+                         
+                       
+                                    
+ 
+
+                                  
+             
+                             
+ 
+
+                           
+             
+               
+                                                         
+                        
+                              
+                     
+                   
+ 
+
+                       
+                     
+                          
+               
+                                                                                
+                                       
+                                    
+                        
+    
+                                  
+                                         
+ 
+
+                                                                                                    
+                                                                                 
+                       
+                        
+                            
+                                                                             
+                                                                
+                                
+                                
+ 
+
+// The full context object analyzer.js's `analyze()` builds and passes as
+// every stage/sql detect()'s second argument, and as the sole argument for
+// 'app' scope (`d.detect(ctx)`). 'config' scope gets a narrower `{ app }`
+// (see auditConfig in analyzer.js), typed per-entry below instead of here.
+//
+// `app` is non-nullable here (unlike `AppModel.app: SparkAppInfo | null`):
+// `analyze()` only ever runs after a full parse, by which point `app` is
+// always populated; the `SparkAppInfo | null` nullability models the
+// in-progress-parse window this file never observes. This matches every
+// 'app'-scope entry below: most defensively write `ctx.app?.foo` anyway
+// (harmless on a non-nullable value), and `coldStart` reads `app.startTime`
+// with no guard at all, which only type-checks if `app` is non-nullable.
+// `auditConfig`'s separate config-scope target type keeps `app` nullable
+// instead, since `auditConfig(appModel.app)` (src/analyzer.js) really can be
+// called with a `null` app and every config-scope entry optional-chains it.
+                              
+                   
+                                     
+                                               
+                                                   
+                                 
+                                    
+                                                
+                                                                         
+                                                                       
+                                                                         
+                                                                 
+                            
+                                             
+ 
+
+// `auditConfig(app)` (src/analyzer.js) calls every 'config'-scope entry's
+// `detect({ app })` directly with whatever `appModel.app` is at call time
+// (`SparkAppInfo | null`), independent of the full `DetectorCtx` above.
+                                       
+                          
+ 
+
+                           
+                                                      
+                                                                  
+                                       
+                                                                                           
+ 
+
+// sparkDoctor SpillPressureDetector (5a) + SpillSkewDetector (5b).
+// Returns { magnitude } | null. Magnitude ∈ 'severe'|'high'|'medium'.
+function computeSpillMagnitude(
+  stage                                                                                                    ,
+  t                 ,
+)                                                     {
+  const { taskCount, spillDiskMax = 0, spillMemMax = 0, spillDiskP50 = 0, spillMemP50 = 0 } = stage;
+  // 5a: absolute volume, branching on task count.
+  if (taskCount === 1) {
+    if (spillDiskMax >= t.singleTaskDiskGiB * GB || spillMemMax >= t.singleTaskMemGiB * GB) return { magnitude: 'severe' };
+  } else {
+    const perTaskDisk = taskCount > 0 ? spillDiskMax / taskCount : 0; // approximation: max used as per-task proxy
+    if (spillDiskMax >= t.highDiskGiB * GB || perTaskDisk >= t.highTaskDiskMB * MB || spillMemMax >= t.highMemGiB * GB) return { magnitude: 'high' };
+    if (spillDiskMax >= t.medDiskMB * MB || spillMemMax >= t.medMemGiB * GB) return { magnitude: 'medium' };
+  }
+  // 5b: ratio+floor skew (requires enough tasks).
+  if (taskCount >= t.skewMinTasks) {
+    if (spillDiskP50 > 0 && spillDiskMax / spillDiskP50 > t.skewRatio && spillDiskMax >= t.skewDiskFloorMB * MB) return { magnitude: 'high' };
+    if (spillMemP50 > 0 && spillMemMax / spillMemP50 > t.skewRatio && spillMemMax >= t.skewMemFloorMB * MB) return { magnitude: 'medium' };
+  }
+  return null;
+}
+
+function pickDominantReason(reasons                                     )                {
+  if (!Array.isArray(reasons) || reasons.length === 0) return null;
+  return [...reasons].sort((a, b) => b.count - a.count)[0].reason;
+}
+
+// Shared by every scope:'sql' detector below. `sql.get(executionId).stageIds`
+// (src/model-assembler.js) is always empty because parser-worker.js never
+// populates it, so stage linkage is derived the other way round, from each
+// stage's own (correctly populated) `sqlExecutionId`. Parameter type is
+// intentionally the minimal shape needed (not `DetectorStage`): callers
+// outside this file (Topbar.tsx, plan-node-detail.ts, plan-graph-model.ts)
+// pass `appModel.stages`, typed `Map<StageId, Stage>` per types.ts, which
+// carries `id`/`sqlExecutionId` but not this file's fuller `DetectorStage`
+// shape.
+export function stageIdsForSqlExec(
+  executionId        ,
+  stages                                                             ,
+)           {
+  const out           = [];
+  for (const s of stages.values()) if (s.sqlExecutionId === executionId) out.push(s.id);
+  return out;
+}
+
+// Shared by the three Plan Advisor detectors below (duplicatePlanSubtree,
+// smallFiles, broadcastSizing): union the given nodes' own `stageIds`, or
+// fall back to the whole execution's stage set when none of them have any
+// coverage. A finding never partially blends a narrowed set with the
+// execution-wide one (see docs/architecture.md's plan-node-to-stage mapping
+// section). `fallback` is a param, not computed here, so callers can compute
+// `stageIdsForSqlExec` once per `detect()` call and reuse it across every
+// finding in that call instead of re-walking `stages` per finding.
+export function unionStageIds(nodes            , fallback          )           {
+  const union = new Set        ();
+  for (const node of nodes) for (const sid of node.stageIds ?? []) union.add(sid);
+  return union.size > 0 ? [...union].sort((a, b) => a - b) : fallback;
+}
+
+// Bottom-up shape computation for duplicate-subtree detection (sparkDoctor
+// idea, src/detectors.js `duplicatePlanSubtree`) and for the cachingOpportunity
+// composite (join/union) detector's anchor-fingerprint contract. `size` is the
+// node count of the subtree rooted at each node; the default fingerprint
+// encodes operator name + sorted metric *names* (never values, per spec) +
+// children fingerprints in original order, so two subtrees with the same shape
+// but different row counts/literals still collide, which is the point (we
+// have no expr/plan/codegen IDs to strip in the first place, since `metrics`
+// never carried them).
+//
+// `opts.includeDetail` (default false) folds `root`'s own `detail` text
+// (normalized via `opts.normalizeDetail`) into ONLY `root`'s fingerprint,
+// never into any recursive child call's fingerprint, regardless of `opts`.
+// This is what lets a caller ask "does this specific node's own detail +
+// structural shape match another node's", without descendant filter/scan
+// detail (literals, paths) ever entering the comparison. The existing
+// `duplicatePlanSubtree` call site passes no options: identical behavior,
+// zero regression to its existing tests.
+                                                          
+
+export function computePlanShapes(
+  root          ,
+  opts                                                                       = {},
+)                                                                  {
+  const { includeDetail = false, normalizeDetail: normalize = (d        ) => d } = opts;
+  const shapeOf = new WeakMap                     ();
+  const allNodes             = [];
+  function visit(node          , isRoot         )            {
+    allNodes.push(node);
+    const childShapes = (node.children ?? []).map((c) => visit(c, false));
+    const size = 1 + childShapes.reduce((sum, c) => sum + c.size, 0);
+    const metricNames = (node.metrics ?? []).map((m) => m.name).sort().join(',');
+    const childFingerprints = childShapes.map((c) => c.fingerprint).join(',');
+    const fingerprint = isRoot && includeDetail
+      ? `${node.name}[${metricNames}]<${normalize(node.detail ?? '')}>{${childFingerprints}}`
+      : `${node.name}[${metricNames}]{${childFingerprints}}`;
+    const shape = { size, fingerprint };
+    shapeOf.set(node, shape);
+    return shape;
+  }
+  visit(root, true);
+  return { shapeOf, allNodes };
+}
+
+// Normalizes an anchor join/union node's own `detail` text for the
+// cachingOpportunity composite detector's structural fingerprint
+// (computePlanShapes's opts.includeDetail path). Strips per-analysis
+// numbering noise that is never part of a join/union's logical identity:
+// expr ids, plan/codegen-stage ids, and AQE's runtime BuildLeft/BuildRight
+// broadcast-side choice (which can flip between executions of the logically
+// identical join based on runtime stats); it then canonicalizes commutative
+// equality operand order so `A.x = B.y` and `B.y = A.x` collide. Everything
+// else (join type, columns, literal values) is kept: that's the
+// semantically meaningful part a leaf-relation-set identity would miss.
+export function normalizeDetail(detail        )         {
+  let s = detail
+    .replace(/#\d+L?/g, '')
+    .replace(/,?\s*plan_id=\d+/g, '')
+    .replace(/\[codegen id\s*:\s*\d+\]/gi, '[codegen id]')
+    .replace(/\bBuild(Left|Right)\b/g, 'BuildSide');
+  s = s.replace(/([A-Za-z_][\w.]*)\s*=\s*([A-Za-z_][\w.]*)/g, (_m, l, r) => {
+    const [a, b] = [l, r].sort();
+    return `${a} = ${b}`;
+  });
+  return s;
+}
+
+const JOIN_NAME_RE = /Join/i;
+
+// Structural operator kind for cachingOpportunity's composite detection:
+// 'join' covers every Spark join physical operator (SortMergeJoin,
+// BroadcastHashJoin, ShuffledHashJoin, BroadcastNestedLoopJoin, the same set
+// plan-summary.js's visitJoin recognizes); 'union' is Spark's exact `Union`
+// node name. CartesianProduct is deliberately excluded (out of scope, same
+// as plan-summary.js's join handling).
+export function planOperatorKind(name        )                          {
+  if (JOIN_NAME_RE.test(name)) return 'join';
+  if (name === 'Union') return 'union';
+  return null;
+}
+
+// Single bottom-up pass over a resolved planTree producing one composite
+// candidate per join/union node, in O(n) total (see the design doc's
+// "Compute cost" section; this deliberately does NOT call
+// computePlanShapes per join/union node, since subtrees overlap and that
+// would be O(n·k) on multi-way star/snowflake joins). For every node it
+// computes the plain (detail-free) fingerprint once (identical formula to
+// computePlanShapes's default path) and merges each subtree's leaf-relation
+// byte map (via scanRelationId, same identity cachingOpportunity's existing
+// leaf aggregation uses) bottom-up. When a node is a join/union, its anchor
+// fingerprint folds only its OWN normalized detail (per computePlanShapes's
+// opts.includeDetail contract), computed here inline, in O(1), from the
+// node's own detail plus its already-computed child fingerprints, not via a
+// nested computePlanShapes call. `ancestorNodes` (strict ancestors, root
+// first) is threaded down for free via the recursion's own call stack, so
+// later nested-composite dedupe (cachingOpportunity.detect()) doesn't need a
+// separate tree walk to determine containment.
+                                     
+                 
+                             
+                      
+                                         
+                            
+ 
+
+export function findCompositeCandidates(root          )                       {
+  const candidates                       = [];
+  const path             = [];
+  function visit(node          )                                                                  {
+    path.push(node);
+    const childResults = (node.children ?? []).map(visit);
+    path.pop();
+
+    const metricNames = (node.metrics ?? []).map((m) => m.name).sort().join(',');
+    const childFingerprints = childResults.map((r) => r.fingerprint).join(',');
+    const fingerprint = `${node.name}[${metricNames}]{${childFingerprints}}`;
+
+    const leafRelationBytes = new Map                ();
+    const rid = scanRelationId(node.name ?? '', node.detail ?? '');
+    if (rid) {
+      const bytesMetric = (node.metrics ?? []).find((m) => m.name === FILES_READ_BYTES);
+      leafRelationBytes.set(rid, (leafRelationBytes.get(rid) ?? 0) + (bytesMetric ? bytesMetric.value : 0));
+    }
+    for (const child of childResults) {
+      for (const [crid, bytes] of child.leafRelationBytes) {
+        leafRelationBytes.set(crid, (leafRelationBytes.get(crid) ?? 0) + bytes);
+      }
+    }
+
+    const operator = planOperatorKind(node.name ?? '');
+    if (operator) {
+      candidates.push({
+        node,
+        operator,
+        fingerprint: `${node.name}[${metricNames}]<${normalizeDetail(node.detail ?? '')}>{${childFingerprints}}`,
+        leafRelationBytes: new Map(leafRelationBytes),
+        ancestorNodes: path.slice(),
+      });
+    }
+
+    return { fingerprint, leafRelationBytes };
+  }
+  visit(root);
+  return candidates;
+}
+
+
+// First scanned table/relation identity found in a subtree (pre-order, per
+// walkPlanTree's contract), or null when the subtree touches no named
+// relation. Surfaced on duplicatePlanSubtree findings (as `sampleRelation`,
+// see findDuplicateSubtrees below) so a user can tell apart two same-shaped
+// duplicate groups that scan different tables (real-log bug: two unrelated
+// "BroadcastExchange over a Project/Filter/Scan" patterns, one per dimension
+// table, produced byte-identical findings). This is best-effort/informational
+// only, not a uniqueness guarantee: it's null for scan-less subtrees (JDBC/
+// Kafka/LocalRelation) and can coincide when two groups share their first-
+// encountered leaf. See findDuplicateSubtrees's groupIndex for the actual
+// discriminator findingId() relies on.
+function firstLeafRelationId(node          )                {
+  let found                = null;
+  walkPlanTree(node, (n) => {
+    if (found) return;
+    found = scanRelationId(n.name ?? '', n.detail ?? '');
+  });
+  return found;
+}
+
+// Groups nodes by fingerprint, keeping only groups of size >= minOccurrences
+// whose shared subtree size is >= minSubtreeSize. De-overlap: processes
+// candidate groups largest-subtree-first, and once a group is accepted every
+// node inside each of its matched occurrences is marked "claimed" so a
+// smaller, fully-nested duplicate group inside an already-accepted match is
+// dropped (a 5-node duplicate should not also emit findings for its 3-node
+// sub-subtrees); occurrences of a smaller group that fall OUTSIDE any
+// accepted larger match still count normally.
+                                 
+                   
+                      
+                      
+                          
+                                
+                     
+                    
+ 
+
+export function findDuplicateSubtrees(
+  root          ,
+  { minSubtreeSize, minOccurrences }                                                    ,
+)                          {
+  const { shapeOf, allNodes } = computePlanShapes(root);
+  const eligible = allNodes.filter(n => shapeOf.get(n) .size >= minSubtreeSize);
+
+  const groups = new Map                    ();
+  for (const n of eligible) {
+    const fp = shapeOf.get(n) .fingerprint;
+    if (!groups.has(fp)) groups.set(fp, []);
+    groups.get(fp) .push(n);
+  }
+
+  const candidates = [...groups.values()]
+    .filter(nodes => nodes.length >= minOccurrences)
+    .sort((a, b) => shapeOf.get(b[0]) .size - shapeOf.get(a[0]) .size);
+
+  const claimed = new WeakSet          ();
+  const markClaimed = (node          ) => {
+    claimed.add(node);
+    for (const c of (node.children ?? [])) markClaimed(c);
+  };
+
+  const results                          = [];
+  for (const nodes of candidates) {
+    const unclaimed = nodes.filter(n => !claimed.has(n));
+    if (unclaimed.length < minOccurrences) continue;
+    for (const n of unclaimed) markClaimed(n);
+    results.push({
+      rootName: unclaimed[0].name,
+      subtreeSize: shapeOf.get(unclaimed[0]) .size,
+      occurrences: unclaimed.length,
+      isExchangeRoot: /Exchange/i.test(unclaimed[0].name),
+      sampleRelation: firstLeafRelationId(unclaimed[0]),
+      // Deterministic position within this execution's group list. `sampleRelation`
+      // is best-effort (null when the subtree touches no named catalog relation, e.g.
+      // JDBC/Kafka/LocalRelation sources, or identical when two groups happen to share
+      // their first-encountered leaf) and is NOT sufficient on its own to guarantee two
+      // structurally-distinct groups get distinct finding ids; groupIndex is the actual
+      // uniqueness guarantee findingId() relies on.
+      groupIndex: results.length,
+      nodes: unclaimed,
+    });
+  }
+  return results;
+}
+
+// Exact metric names Spark emits: verified against a real SQLExecutionStart
+// event's sparkPlanInfo in examples/big-application_1777489669889_51251_1.
+// NOTE: the write-side byte metric is "written output", not "size of written
+// files" as an earlier draft of this detector's spec assumed.
+const FILES_READ_COUNT = 'number of files read';
+const FILES_READ_BYTES = 'size of files read';
+const FILES_WRITTEN_COUNT = 'number of written files';
+const FILES_WRITTEN_BYTES = 'written output';
+
+// Byte size of a join-side subtree for broadcast sizing (dataflint
+// JoinToBroadcastAlert). Stops descending the instant a node carries a
+// "data size" metric: that node's value already aggregates everything
+// beneath it (e.g. an Exchange's "data size" already reflects everything it
+// shuffled), so summing further down would double-count. Only recurses into
+// children when the current node carries no such metric.
+function sumBoundarySize(node          )         {
+  const m = (node.metrics ?? []).find(x => x.name === 'data size');
+  if (m) return m.value;
+  let sum = 0;
+  for (const c of (node.children ?? [])) sum += sumBoundarySize(c);
+  return sum;
+}
+
+// Nodes that actually fed sumBoundarySize's total for this subtree: mirrors
+// its recursion exactly (same "data size" metric check, same stop condition)
+// so the implicated stageIds line up with the size that was actually
+// compared, however deep that turns out to live.
+function boundarySizeContributors(node          )             {
+  const m = (node.metrics ?? []).find((x) => x.name === 'data size');
+  if (m) return [node];
+  const out             = [];
+  for (const c of (node.children ?? [])) out.push(...boundarySizeContributors(c));
+  return out;
+}
+
+// Max/median deviation ratio over a list of {key, value}; returns the
+// exceeding entry's key + ratio, or null when < 3 samples or median 0.
+function maxMedianRatio(
+  samples                                  ,
+)                                                       {
+  if (samples.length < 3) return null;
+  const vals = samples.map(s => s.value).sort((a, b) => a - b);
+  const median = vals[Math.floor(vals.length / 2)];
+  if (median <= 0) return null;
+  const top = samples.reduce((a, b) => (b.value > a.value ? b : a));
+  return { key: top.key, ratio: top.value / median, value: top.value };
+}
+
+                                       
+               
+                  
+                                            
+                                                 
+                     
+ 
+
+// Machine-readable detector metadata for the evidence report: type, version,
+// scope, thresholds, and doc anchor per entry (no `detect` closure). Lets a
+// portable report record exactly which detector + threshold set produced each
+// finding, so evidence stays reproducible as detectors evolve.
+export function detectorCatalog()                         {
+  return DETECTORS.map((d) => ({
+    type: d.type,
+    version: d.version ?? 1,
+    scope: d.scope,
+    thresholds: d.thresholds,
+    docAnchor: d.docAnchor,
+  }));
+}
+
+// True task-duration skew ratio for a stage: P95/median once there are enough
+// tasks to trust a P95 estimate, otherwise max/median. Returns null when the
+// stage has no measurable median (p50 === 0). Exported so the CLI budget gate
+// (src/cli/budgets.ts) can recompute the same ratio the detector uses, rather
+// than reading the detector's own findings (which are floored at ratioWarn).
+//
+// Fields widened to optional: the `skew` detector below only ever calls this
+// with an already-finalized `DetectorStage` (all four always numeric by the
+// time `analyze()` runs), but `cli/budgets.ts`'s `checkSkew` calls it
+// directly against `AppModel.stages`: real `Stage` records for a stage that
+// never received a `StageCompleted` event (e.g. an unfinished run) genuinely
+// lack these fields (see `types.ts`'s `Stage`). The `as number` casts below
+// preserve the original behavior byte-for-byte: dividing through an absent
+// field still naturally produces `NaN` (as it always has for untyped JS
+// callers), rather than adding a new guard that would change the result.
+export function computeSkewRatio(
+  stage                                                                                                      ,
+  minTasksForP95        ,
+)                                                                {
+  const { taskCount, taskDurationP50: p50, taskDurationP95: p95, taskDurationMax: max } = stage;
+  if (p50 === 0) return null;
+  return (taskCount ?? 0) >= minTasksForP95
+    ? { ratio: (p95          ) / (p50          ), metric: 'P95/median' }
+    : { ratio: (max          ) / (p50          ), metric: 'max/median' };
+}
+
+// Absolute-magnitude floor expressed as a % of total app runtime rather than
+// a fixed ms constant (mirrors computeSpillMagnitude's/slowHost's ratio+floor
+// pattern above/below): a skew/straggler/taskStageSkew ratio computed on a
+// handful of milliseconds is noise in a run that took hours, but the same
+// absolute waste is real in a run that only took seconds; a fixed-ms floor
+// can't scale between those. Used by the `skew` and `straggler` entries
+// below, each gated via `clippedWasteMs` (below) against the *same
+// occupancy-clipped* wall-clock figure
+// src/impact-estimator.ts displays as that finding's savings, not the raw
+// pre-clip delta, which can stay large after clipping collapses the
+// recoverable time to near zero (the stage's own longest task already
+// accounts for nearly all of its wall-clock window).
+// NOT SOURCED: floor percentages are our own noise floor, unvalidated.
+function computeAppDurationMs(ctx              )                {
+  const app = ctx?.app;
+  if (app?.startTime == null || app?.endTime == null) return null;
+  const durationMs = app.endTime - app.startTime;
+  return durationMs > 0 ? durationMs : null;
+}
+
+// Unknown app timing (computeAppDurationMs returned null) never suppresses a
+// finding; it just skips the floor gate, preserving prior ratio-only
+// behavior when total runtime can't be computed.
+function meetsRuntimeFloor(wasteMs        , appDurationMs               , floorPct        )          {
+  return appDurationMs == null || wasteMs >= appDurationMs * floorPct;
+}
+
+// Runs a raw waste delta through the same ceiling/occupancy clip
+// src/impact-estimator.ts's singleStageImpact applies before display, so the
+// runtime floor above is checked against a stage's actual recoverable
+// wall-clock time rather than a delta that a physical floor (the stage's own
+// longest task) may leave almost entirely unrecoverable. Falls back to the
+// raw delta when occupancy data isn't available for this stage (ctx omitted,
+// or the stage was excluded from the occupancy sweep for having <= 0
+// duration), same as the pre-existing ratio-only behavior for unknown app
+// timing above.
+function clippedWasteMs(wasteMs        , stageId        , ctx              )         {
+  if (!ctx) return wasteMs;
+  const est = estimateSingleStage(wasteMs, stageId, ctx.stages                                          , ctx.occupancy);
+  return est ? est.wallClock.high : wasteMs;
+}
+
+// cacheUtilization's per-RDD copy (private to this file), following the
+// existing pattern of small per-detector formatting helpers (e.g.
+// pickDominantReason above). Both variants share the same confidence/
+// validationRequired text: the ratio is a point-in-time storage snapshot
+// from stage-submission events (src/event-handlers.js's mergeStageRddInfo),
+// not a runtime block-access read-count.
+const CACHE_UTILIZATION_VALIDATION =
+  "This ratio is a point-in-time storage snapshot from stage-submission events, not a runtime read-count. Confirm against the Spark UI's Storage tab before acting.";
+
+function partialCacheFinding(rdd                 , cachedRatio        , impactBand                    )          {
+  const rddName = rdd.name || `RDD ${rdd.id}`;
+  const cachedPct = Math.round(cachedRatio * 100);
+  const evictedPct = 100 - cachedPct;
+  return {
+    type: 'cacheUtilization', variant: 'partialCache', stageId: null,
+    rddId: rdd.id, rddName,
+    impactBand, metric: 'cachedRatio', value: cachedPct,
+    confidence: 'medium',
+    validationRequired: CACHE_UTILIZATION_VALIDATION,
+    memorySize: rdd.memorySize, diskSize: rdd.diskSize,
+    numCachedPartitions: rdd.numCachedPartitions, numPartitions: rdd.numPartitions,
+    recommendation: `RDD ${rddName} is ${evictedPct}% evicted from cache (${cachedPct}% of partitions cached). Increase executor memory or reduce the cached dataset size.`,
+  };
+}
+
+function diskSpilloverFinding(rdd                 , diskRatio        , impactBand                    )          {
+  const rddName = rdd.name || `RDD ${rdd.id}`;
+  const diskPct = Math.round(diskRatio * 100);
+  return {
+    type: 'cacheUtilization', variant: 'diskSpillover', stageId: null,
+    rddId: rdd.id, rddName,
+    impactBand, metric: 'diskRatio', value: diskPct,
+    confidence: 'medium',
+    validationRequired: CACHE_UTILIZATION_VALIDATION,
+    memorySize: rdd.memorySize, diskSize: rdd.diskSize,
+    numCachedPartitions: rdd.numCachedPartitions, numPartitions: rdd.numPartitions,
+    recommendation: `RDD ${rddName} is ${diskPct}% spilled to disk despite requesting MEMORY_AND_DISK. Executor memory may be too small for this cached dataset.`,
+  };
+}
+
+// Entry shape for every item in DETECTORS. `TTarget` stays `unknown` at the
+// array level (kept as the default, never instantiated per-scope): `detect`'s
+// real first-argument shape varies by `scope` (a `DetectorStage` for
+// 'stage', a `DetectorSqlExec` for 'sql', a `DetectorCtx` for 'app', or a
+// narrower `{ app }` for 'config'; see analyzer.js's `analyze`/`auditConfig`,
+// the only real callers), and unifying those four into one `TTarget` would
+// need either an unsound cast or a discriminated-union-of-detectors redesign
+// this migration task doesn't ask for. Each entry below still gets a
+// precisely-typed `detect` by annotating its own `target`/`ctx` parameters
+// directly: object-literal methods (this `detect(target) {}` shorthand, not
+// an arrow function assigned to a property) are checked bivariantly against
+// an interface's method parameter types, so a narrower, concrete annotation
+// here does not conflict with `Detector`'s `unknown` declaration.
+                                              
+               
+                                            
+                
+                       
+                  
+                     
+                                                                           
+                                                                            
+                                                                             
+                                                                        
+                    
+                                                 
+                          
+                      
+                        
+                    
+                                                               
+                                                                     
+ 
+
+// Threshold field naming convention:
+// *Pct = 0–1 fraction (normalized)
+// *Pct100 = 0–100 scale
+// *Ratio = multiplicative factor
+// *Share/*Rate/*Util = 0–1 fraction (normalized)
+
+// `straggler`'s own noise-floor thresholds (NOT SOURCED: unvalidated),
+// exported so src/impact-band.ts can reuse the same figures as its global
+// impact-band floor instead of hand-copying the literals.
+export const STRAGGLER_FLOOR_PCT_WARN = 0.005;
+export const STRAGGLER_FLOOR_PCT_CRIT = 0.02;
+
+export const DETECTORS             = [
+  {
+    type: 'skew', scope: 'stage', order: 30, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-skew',
+    thresholds: { ratioWarn: 3, minTasksForP95: 20, floorPctWarn: 0.005 },
+    detect(
+             
+                                                                                        
+        
+      stage               ,
+      ctx              ,
+    )                 {
+      const result = computeSkewRatio(stage, this.thresholds.minTasksForP95);
+      if (result === null) return null;
+      const { ratio, metric } = result;
+      if (ratio <= this.thresholds.ratioWarn) return null;
+      // Same absolute delta src/impact-estimator.ts's 'skew' case reports as
+      // this finding's savings; clipped the same way before the floor check
+      // so the gate agrees with what's actually displayed.
+      const wasteMs = Math.max(0, metric === 'P95/median' ? stage.taskDurationP95 - stage.taskDurationP50 : stage.taskDurationMax - stage.taskDurationP50);
+      const appDurationMs = computeAppDurationMs(ctx);
+      const floorWasteMs = clippedWasteMs(wasteMs, stage.id, ctx);
+      if (!meetsRuntimeFloor(floorWasteMs, appDurationMs, this.thresholds.floorPctWarn)) return null;
+      const value = Math.round(ratio * 10) / 10;
+      return {
+        type: 'skew', stageId: stage.id,
+        impactBand: 'warning',
+        metric, value,
+        recommendation: `Task duration ratio (${metric}) is ${value}×: for join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key to reduce task skew.`,
+      };
+    },
+  },
+  {
+    type: 'stageShape', scope: 'stage', order: 35, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-stage-shape',
+    thresholds: { pRatioMax: 0.5, oiRatioMax: 10, skewWarn: 3 },
+    detect(
+                                                                                        
+      stage               ,
+      ctx              ,
+    )            {
+      const out            = [];
+      const execCount = (stage.executorStats ?? []).length;
+      const cores = ctx?.app?.resources?.executor?.cores ?? 1;
+      const totalCores = execCount * cores;
+      // PRatio: under-parallelization.
+      if (totalCores > 0) {
+        const pRatio = stage.taskCount / totalCores;
+        if (pRatio < this.thresholds.pRatioMax) {
+          out.push({
+            type: 'stageShape', stageId: stage.id, impactBand: 'info',
+            rule: 'lowParallelism', metric: 'pRatio', value: Math.round(pRatio * 100) / 100,
+            // Absolute core count behind pRatio, for the impact estimator's idle-core-ms figure.
+            totalCores,
+            recommendation: `This stage runs ${stage.taskCount} ${stage.taskCount === 1 ? 'task' : 'tasks'} across ~${totalCores} cores, so it is under-parallelized and leaves cluster capacity idle.`,
+          });
+        }
+      }
+      // OIRatio: data explosion. Skip when inputBytes is 0 (Infinity guard).
+      if (stage.inputBytes > 0) {
+        const oiRatio = stage.outputBytes / stage.inputBytes;
+        if (oiRatio > this.thresholds.oiRatioMax) {
+          out.push({
+            type: 'stageShape', stageId: stage.id, impactBand: 'info',
+            rule: 'dataExplosion', metric: 'oiRatio', value: Math.round(oiRatio * 10) / 10,
+            recommendation: `This stage outputs ${Math.round(oiRatio)}× its input volume: check for an exploding join or a cross product.`,
+          });
+        }
+      }
+      // TaskStageSkew: straggler cost vs stage wall-clock. Skip near-zero duration.
+      // Always info, like its lowParallelism/dataExplosion siblings above: satisfying
+      // this trigger mathematically forces the occupancy-clipped wall-clock estimate to
+      // exactly zero on every firing (see impact-estimator.ts's costOnly branch below),
+      // so there is no wall-clock-backed impact-band tier left to gate on.
+      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+      if (stageDurationMs > 0) {
+        const ratio = stage.taskDurationMax / stageDurationMs;
+        if (ratio > this.thresholds.skewWarn) {
+          out.push({
+            type: 'stageShape', stageId: stage.id, impactBand: 'info',
+            rule: 'taskStageSkew', metric: 'taskStageSkew', value: Math.round(ratio * 10) / 10,
+            // Absolute core count, for the impact estimator's idle-core-ms figure.
+            totalCores,
+            recommendation: `One task takes ${Math.round(ratio * 10) / 10}× this stage's wall-clock duration; a single straggler is gating the whole stage.`,
+          });
+        }
+      }
+      return out;
+    },
+  },
+  {
+    type: 'shuffle', scope: 'stage', order: 20, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-shuffle',
+    thresholds: { minBytes: 50 * MB },
+    detect(
+                                                 
+      stage               ,
+    )                 {
+      const bytes = stage.shuffleReadBytes;
+      if (bytes <= this.thresholds.minBytes) return null;
+      return {
+        type: 'shuffle', stageId: stage.id,
+        impactBand: 'info',
+        metric: 'shuffleReadBytes', value: bytes,
+        recommendation: `${formatBytes(bytes)} shuffled in this stage: consider increasing spark.sql.shuffle.partitions or adding a broadcast join.`,
+      };
+    },
+  },
+  {
+    type: 'partitionSizing', scope: 'stage', order: 22, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-shuffle',
+    thresholds: { skewRatio: 5, skewFloorBytes: 256 * MB, lowParTotalBytes: GB, lowParMaxTasks: 7, maxPartBytes: 5 * GB },
+    detect(
+             
+                     
+                                                                              
+                                                       
+          
+        
+      stage               ,
+    )            {
+      const out            = [];
+      const { shuffleReadP50: p50, shuffleReadMax: max, shuffleReadBytes: total, taskCount } = stage;
+      if (max > this.thresholds.skewRatio * p50 && max > this.thresholds.skewFloorBytes) {
+        // p50 can be 0 (more than half the shuffle partitions empty): a
+        // ratio against zero renders the literal string "Infinity×", so fall
+        // back to median-free phrasing instead of dividing by p50.
+        const ratioText = p50 > 0
+          ? `${Math.round(max / p50 * 10) / 10}× the median (${formatBytes(p50)})`
+          : `far larger than the median (${formatBytes(p50)}, effectively empty)`;
+        out.push({
+          type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
+          rule: 'shufflePartitionSkew', metric: 'shuffleReadMax', value: max,
+          recommendation: `The largest shuffle partition (${formatBytes(max)}) is ${ratioText}: for join skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.`,
+        });
+      }
+      if (total >= this.thresholds.lowParTotalBytes && taskCount <= this.thresholds.lowParMaxTasks) {
+        out.push({
+          type: 'partitionSizing', stageId: stage.id, impactBand: 'warning',
+          rule: 'lowShuffleParallelism', metric: 'taskCount', value: taskCount,
+          recommendation: `${Math.round(total / GB * 10) / 10} GB of shuffle spread over only ${taskCount} tasks: raise spark.sql.shuffle.partitions so each partition is smaller.`,
+        });
+      }
+      if (max >= this.thresholds.maxPartBytes) {
+        out.push({
+          type: 'partitionSizing', stageId: stage.id, impactBand: 'critical',
+          rule: 'maxPartitionTooBig', metric: 'shuffleReadMax', value: max,
+          recommendation: `A single shuffle partition (${formatBytes(max)}) exceeds 5 GB: this will OOM or spill heavily. Repartition to break it up before this stage.`,
+        });
+      }
+      return out;
+    },
+  },
+  {
+    type: 'spill', scope: 'stage', order: 10, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-spill',
+    thresholds: { singleTaskDiskGiB: 1, singleTaskMemGiB: 4, highDiskGiB: 1, highTaskDiskMB: 512, highMemGiB: 4, medDiskMB: 256, medMemGiB: 1, skewRatio: 5, skewDiskFloorMB: 128, skewMemFloorMB: 256, skewMinTasks: 10 },
+    detect(                                       stage               )                 {
+      if (stage.memoryBytesSpilled === 0) return null;
+      const cls = stage.spillClassification;
+      const classified = cls === 'skew' || cls === 'volume';
+      const mag = computeSpillMagnitude(stage, this.thresholds);
+      const impactBand = 'warning';
+      return {
+        type: 'spill', stageId: stage.id, impactBand,
+        spillMagnitude: mag?.magnitude,
+        metric: 'memoryBytesSpilled', value: stage.memoryBytesSpilled,
+        confidence: classified ? 'medium' : 'low',
+        validationRequired: classified
+          ? 'Spill cause is inferred from the share of tasks that spilled: confirm against per-task spill metrics in the Spark UI.'
+          : 'Spill cause could not be classified: inspect per-task spill metrics in the Spark UI before acting.',
+        recommendation: cls === 'skew'
+          ? `${formatBytes(stage.memoryBytesSpilled)} spilled, skew-driven: fix task skew first; adding memory will not help.`
+          : `${formatBytes(stage.memoryBytesSpilled)} spilled: raise spark.sql.shuffle.partitions or increase executor memory.`,
+      };
+    },
+  },
+  {
+    type: 'gc', scope: 'stage', order: 50, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-gc',
+    thresholds: {
+      warnPct100: 10,
+      // Descending tier: Dr. Elephant ExecutorGcHeuristic, ported as-is.
+      lowInfoPct100: 5,
+      // NOT SOURCED: our own noise floor so a stage that barely ran (gcPct
+      // near 0 or wildly inflated by a tiny denominator) does not flag,
+      // in either direction.
+      minRunTimeMs: 10000,
+    },
+    detect(
+             
+                                                                                        
+        
+      stage               ,
+    )                 {
+      const pct = stage.gcPct;
+      if ((stage.executorRunTime ?? 0) >= this.thresholds.minRunTimeMs
+          && pct > this.thresholds.warnPct100) {
+        const value = Math.round(pct * 10) / 10;
+        return {
+          type: 'gc', stageId: stage.id,
+          impactBand: 'warning',
+          metric: 'gcPct', value,
+          recommendation: `GC consumed ${value}% of executor run time: reduce object creation, use primitive types, avoid UDFs, increase executor memory.`,
+        };
+      }
+      // Low-GC (cost) branch: only for stages that ran long enough to be meaningful.
+      if ((stage.executorRunTime ?? 0) >= this.thresholds.minRunTimeMs
+          && pct < this.thresholds.lowInfoPct100) {
+        const value = Math.round(pct * 10) / 10;
+        return {
+          type: 'gc', stageId: stage.id, direction: 'low',
+          impactBand: 'info',
+          metric: 'gcPct', value,
+          recommendation: `GC consumed only ${value}% of executor run time: memory may be over-provisioned; consider reducing spark.executor.memory for cost savings.`,
+        };
+      }
+      return null;
+    },
+  },
+  {
+    type: 'slowHost', scope: 'stage', order: 60, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-slow-host',
+    thresholds: {
+      minHosts: 3, minTasks: 15, ratioWarn: 2.0, minShare: 0.20, shareWarn: 0.75, taskShareWarn: 0.50, ratioTiers: [1.33, 1.78, 3.16, 10],
+      // Absolute-magnitude floors (mirrors computeSpillMagnitude's ratio+floor
+      // pattern): on short stages, sub-second/sub-64MB differences between
+      // hosts or executors produce huge ratios that are pure noise, not a
+      // real slow-host problem. 1s is well above typical per-task scheduling
+      // jitter but well below the tens-of-seconds+ means genuine slow-host
+      // stages exhibit; 64MB mirrors the spill detector's disk-skew floor.
+      floorMs: 1000, floorBytes: 64 * MB,
+    },
+    detect(
+             
+                     
+                                                                                  
+                                                                                                              
+          
+        
+      stage               ,
+    )                   {
+      const hosts = stage.hostStats ?? [];
+      const execs0 = stage.executorStats ?? [];
+      if ((hosts.length < this.thresholds.minHosts && execs0.length < this.thresholds.minHosts) || stage.taskCount < this.thresholds.minTasks) return null;
+      const out            = [];
+      if (hosts.length >= this.thresholds.minHosts) {
+        const means = hosts.map(h => ({ host: h.host, taskCount: h.taskCount, mean: h.totalDuration / h.taskCount }));
+        const sorted = [...means].map(h => h.mean).sort((a, b) => a - b);
+        const overallMedian = sorted[Math.floor(sorted.length / 2)];
+        if (overallMedian > 0) {
+          for (const h of means) {
+            const ratio = h.mean / overallMedian;
+            const share = h.taskCount / stage.taskCount;
+            if (ratio < this.thresholds.ratioWarn || share < this.thresholds.minShare || h.mean < this.thresholds.floorMs) continue;
+            out.push({
+              type: 'slowHost', stageId: stage.id,
+              impactBand: 'warning',
+              metric: 'hostMeanRatio', value: Math.round(ratio * 10) / 10,
+              // `value` is a ratio; the estimator needs the absolute per-host mean.
+              hostMeanMs: h.mean,
+              host: h.host, hostTaskShare: Math.round(share * 100) / 100,
+              recommendation: `Check executor logs for ${h.host}: possible bad node, disk pressure, or NUMA misalignment.`,
+            });
+          }
+        }
+        const totalDuration = hosts.reduce((s, h) => s + h.totalDuration, 0);
+        if (totalDuration > 0) {
+          for (const h of hosts) {
+            const durationShare = h.totalDuration / totalDuration;
+            const taskShare = h.taskCount / stage.taskCount;
+            if (durationShare >= this.thresholds.shareWarn && taskShare >= this.thresholds.taskShareWarn) {
+              out.push({
+                type: 'slowHost', stageId: stage.id, impactBand: 'warning',
+                variant: 'durationShare',
+                metric: 'hostDurationShare', value: Math.round(durationShare * 100) / 100,
+                // `value` is a 0-1 share; the estimator needs the absolute per-host mean.
+                hostMeanMs: h.totalDuration / h.taskCount,
+                host: h.host, hostTaskShare: Math.round(taskShare * 100) / 100,
+                recommendation: `${h.host} is doing ${Math.round(durationShare * 100)}% of this stage's total task time: check for data locality or partition assignment skewing work onto one node.`,
+              });
+            }
+          }
+        }
+      }
+      const execs = stage.executorStats ?? [];
+      const tiers = this.thresholds.ratioTiers;
+      const impactBandFor = (r        )                                         =>
+        r >= tiers[3] ? 'critical' : (r >= tiers[1] ? 'warning' : (r >= tiers[0] ? 'info' : null));
+      const floorMs = this.thresholds.floorMs, floorBytes = this.thresholds.floorBytes;
+      const dims                                                                                    = [
+        { dimension: 'taskTime', floor: floorMs, samples: execs.filter(e => e.taskCount > 0).map(e => ({ key: e.executorId, value: e.totalDuration / e.taskCount })) },
+        { dimension: 'inputBytes', floor: floorBytes, samples: execs.map(e => ({ key: e.executorId, value: e.inputBytes ?? 0 })) },
+        { dimension: 'shuffleBytes', floor: floorBytes, samples: execs.map(e => ({ key: e.executorId, value: (e.shuffleReadBytes ?? 0) + (e.shuffleWriteBytes ?? 0) })) },
+      ];
+      // Storage-memory dimension: best-effort, only when executorMetrics present.
+      const em = stage.executorMetrics instanceof Map ? stage.executorMetrics : null;
+      if (em && em.size >= 3) {
+        dims.push({ dimension: 'storageMemory', floor: floorBytes, samples: [...em.entries()].map(([id, m]) => ({ key: id, value: (m.onHeapStorageMemory ?? 0) + (m.offHeapStorageMemory ?? 0) })) });
+      }
+      for (const d of dims) {
+        const r = maxMedianRatio(d.samples);
+        if (!r || r.value < d.floor) continue; // absolute-magnitude floor: same ratio+floor shape as computeSpillMagnitude
+        const tier = impactBandFor(r.ratio);
+        if (!tier) continue;
+        // taskTime is the only wallClock-bearing dimension here: fixed fallback
+        // (its current floor case), overwritten by deriveImpactBand whenever this
+        // finding gets a real wallClock estimate. The other three dimensions never
+        // get a wallClock estimate, so they keep the dynamic tier unchanged.
+        const impactBand = d.dimension === 'taskTime' ? 'info' : tier;
+        out.push({
+          type: 'slowHost', stageId: stage.id, impactBand,
+          variant: 'multiDim', dimension: d.dimension,
+          metric: 'execMaxMedianRatio', value: Math.round(r.ratio * 10) / 10,
+          // `value` is a ratio; `execMaxValue` is the deviating sample's raw magnitude
+          // in this dimension's own unit (ms for taskTime, bytes for the rest).
+          execMaxValue: r.value,
+          executorId: r.key,
+          recommendation: `Executor ${r.key} deviates ${Math.round(r.ratio * 10) / 10}× from the median on ${d.dimension}: investigate uneven partition assignment or a degraded executor.`,
+        });
+      }
+      return out;
+    },
+  },
+  {
+    type: 'stageSlowness', scope: 'stage', order: 65, fixEffort: 'code', version: 2,
+    docAnchor: '#bottleneck-stage-slowness',
+    thresholds: { infoMin: 15 },
+    // Cross-detector suppression (new pattern; see "Detector contract" in
+    // docs-site/contributor-guide/architecture/detector-contract.md).
+    // Requires this entry to be declared AFTER slowHost in DETECTORS so
+    // slowHost findings are already in `out`.
+    suppressWhen(finding, out) {
+      return out.some(o => o.type === 'slowHost' && o.stageId === finding.stageId);
+    },
+    detect(
+                                                
+      stage               ,
+    )                 {
+      // Basis is real wall-clock stage duration, not per-executor average
+      // (Decision 7); Task 11's impact-estimator formula reuses this exact
+      // stageDurationMs computation.
+      const stageDurationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+      if (!(stageDurationMs > 0)) return null;
+      const durationMinutes = stageDurationMs / 60000;
+      const t = this.thresholds;
+      const impactBand = durationMinutes >= t.infoMin ? 'info' : null;
+      if (!impactBand) return null;
+      const value = Math.round(durationMinutes * 10) / 10;
+      return {
+        type: 'stageSlowness', stageId: stage.id, impactBand,
+        metric: 'stageDurationMinutes', value,
+        recommendation: `This stage ran ${value} minutes with no more specific cause flagged: profile its query plan and check for wide shuffles or expensive UDFs.`,
+      };
+    },
+  },
+  {
+    type: 'stageFailed', scope: 'stage', order: 42, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-failures',
+    thresholds: {},
+    detect(stage               )                 {
+      if (stage.stageFailureReason == null) return null;
+      return {
+        type: 'stageFailed', stageId: stage.id, impactBand: 'critical',
+        variant: 'stageFailure',
+        metric: 'stageFailureReason', value: stage.stageFailureReason,
+        recommendation: `This stage attempt failed outright. Inspect the driver log for the failure reason and the job that triggered it.`,
+      };
+    },
+  },
+  {
+    type: 'failures', scope: 'stage', order: 40, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-failures',
+    thresholds: { minTasks: 10, warnRate: 0.05, critRate: 0.20 },
+    detect(
+                                                                                     
+      stage               ,
+    )                 {
+      if (stage.taskCount < this.thresholds.minTasks) return null;
+      if (!stage.failedTasks) return null;
+      const failureRate = stage.failedTasks / stage.taskCount;
+      if (failureRate <= this.thresholds.warnRate) return null;
+      const value = Math.round(failureRate * 1000) / 10;
+      const dominantReason = pickDominantReason(stage.failureReasons);
+      return {
+        type: 'failures', stageId: stage.id,
+        impactBand: failureRate > this.thresholds.critRate ? 'critical' : 'warning',
+        metric: 'failureRate', value,
+        failedTasks: stage.failedTasks,
+        dominantReason,
+        recommendation: `${value}% of tasks failed${dominantReason ? ` (dominant reason: ${dominantReason})` : ''}: investigate driver logs for executor instability or data-driven errors.`,
+      };
+    },
+  },
+  {
+    type: 'straggler', scope: 'stage', order: 70, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-straggler',
+    // floorPctWarn/floorPctCrit are re-exported as STRAGGLER_FLOOR_PCT_WARN/CRIT
+    // below and reused as src/impact-band.ts's global noise floor: keep the two
+    // in sync, don't hand-edit one without the other.
+    thresholds: { minTasks: 10, shareWarn: 0.05, warnPct: 0.10, critPct: 0.20, floorPctWarn: STRAGGLER_FLOOR_PCT_WARN, floorPctCrit: STRAGGLER_FLOOR_PCT_CRIT },
+    detect(
+             
+                                                                                                                                          
+        
+      stage               ,
+      ctx              ,
+    )                 {
+      if (stage.taskCount < this.thresholds.minTasks) return null;
+      const stragglerShare = (stage.stragglerCount ?? 0) / stage.taskCount;
+      if ((stage.speculativeTasks ?? 0) === 0 && stragglerShare <= this.thresholds.shareWarn) return null;
+      const useSpeculative = (stage.speculativeTasks ?? 0) > 0;
+      const speculativeShare = useSpeculative ? stage.speculativeTasks / stage.taskCount : 0;
+      // Same absolute delta src/impact-estimator.ts's shared straggler/
+      // stageShape case reports as this finding's savings: a high share of
+      // stragglers/speculative retries on a stage whose tasks barely vary in
+      // duration models near-zero savings, so it must not outrank 'info'.
+      // Clipped the same way before the floor check so the gate agrees with
+      // what's actually displayed.
+      const wasteMs = Math.max(0, stage.taskDurationMax - stage.taskDurationP50);
+      const appDurationMs = computeAppDurationMs(ctx);
+      const floorWasteMs = clippedWasteMs(wasteMs, stage.id, ctx);
+      const meetsWarnFloor = meetsRuntimeFloor(floorWasteMs, appDurationMs, this.thresholds.floorPctWarn);
+      const meetsCritFloor = meetsRuntimeFloor(floorWasteMs, appDurationMs, this.thresholds.floorPctCrit);
+      const speculativeTier = speculativeShare >= this.thresholds.critPct && meetsCritFloor ? 'critical'
+                             : speculativeShare >= this.thresholds.warnPct && meetsWarnFloor ? 'warning' : 'info';
+      // Straggler share has no dedicated critical tier per
+      // docs-site/contributor-guide/architecture/detector-contract.md; it can only push to warning.
+      const stragglerTier = stragglerShare > this.thresholds.shareWarn && meetsWarnFloor ? 'warning' : 'info';
+      // Fixed fallback: overwritten by deriveImpactBand (src/impact-band.ts) whenever
+      // this finding gets a real wallClock estimate, which is the common case. Only
+      // surfaces on the rare miss (stage excluded from the occupancy sweep).
+      const impactBand = 'info';
+      // Report whichever signal actually drove the finding, not just whether
+      // speculative execution happened to be on: a high stragglerShare with
+      // few/no speculative retries must not be reported as a low-value
+      // speculativeTasks count (real-log bug: a 50%-straggler-share stage
+      // with 1 speculative task reported an impact band of 'warning' but metric
+      // 'speculativeTasks: 1', hiding the actual cause). Ties keep the prior
+      // default (speculative-driven) so existing speculative-only findings
+      // are unaffected.
+      const useSpeculativeMetric = useSpeculative && !(IMPACT_BAND_ORDER[stragglerTier] < IMPACT_BAND_ORDER[speculativeTier]);
+      const value = useSpeculativeMetric ? stage.speculativeTasks : Math.round(stragglerShare * 100);
+      const detail = useSpeculativeMetric
+        ? `${value} speculative attempt${value === 1 ? '' : 's'} discarded`
+        : `${value}% of tasks straggled`;
+      return {
+        type: 'straggler', stageId: stage.id, impactBand,
+        metric: useSpeculativeMetric ? 'speculativeTasks' : 'stragglerShare',
+        value,
+        unit: useSpeculativeMetric ? 'count' : 'pct',
+        speculativeTasks: stage.speculativeTasks ?? 0,
+        stragglerCount: stage.stragglerCount ?? 0,
+        recommendation: `${detail}: investigate stragglers, likely candidates for AQE skewJoin or data locality issues.`,
+      };
+    },
+  },
+  {
+    type: 'speculationWaste', scope: 'stage', order: 71, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-straggler', confidence: 'low',
+    thresholds: { minWasted: 5, minWasteMs: 60000 },
+    detect(
+             
+                                                              
+                           
+        
+      stage               ,
+    )                 {
+      const wasted = stage.speculationWastedAttempts ?? 0;
+      const wastedMs = stage.speculationWasteMs ?? 0;
+      if (wasted < this.thresholds.minWasted || wastedMs < this.thresholds.minWasteMs) return null;
+      return {
+        type: 'speculationWaste', stageId: stage.id,
+        impactBand: 'warning',
+        metric: 'speculationWasteMs', value: wastedMs,
+        confidence: this.confidence,
+        recommendation: `Speculative execution discarded ${Math.round(wastedMs / 1000)}s of executor time in this stage; if task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.`,
+      };
+    },
+  },
+  {
+    type: 'retryWaste', scope: 'stage', order: 45, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-retry-waste',
+    thresholds: { minWasted: 3, minWasteMs: 30000 },
+    detect(
+                                                                      
+      stage               ,
+    )                 {
+      const wasted = stage.wastedAttempts ?? 0;
+      const wastedMs = stage.retryWasteMs ?? 0;
+      if (wasted < this.thresholds.minWasted || wastedMs < this.thresholds.minWasteMs) return null;
+      return {
+        type: 'retryWaste', stageId: stage.id,
+        impactBand: 'warning',
+        metric: 'retryWasteMs', value: wastedMs,
+        recommendation: `Retried task attempts wasted ${Math.round(wastedMs / 1000)}s of executor time (${wasted} attempt${wasted === 1 ? '' : 's'}) even though the stage completed: investigate executor loss or fetch failures.`,
+        extended: `${wasted} task attempts were superseded by a later retry, wasting ${Math.round(wastedMs / 1000)}s of executor time. Common causes: executor loss (OOM-kill, node death) or shuffle FetchFailed forcing a stage-map recompute. Check driver logs for the dominant reason (see the Failures widget) even if the final failure rate looks low; retries hide the true cost.`,
+      };
+    },
+  },
+  {
+    type: 'tinyTask', scope: 'stage', order: 80, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-tiny-tasks',
+    thresholds: { minTasks: 100, maxP50: 500, maxP95: 1000 },
+    detect(
+                                                                                 
+      stage               ,
+    )                 {
+      if (stage.taskCount < this.thresholds.minTasks) return null;
+      if (stage.taskDurationP50 > this.thresholds.maxP50 || stage.taskDurationP95 > this.thresholds.maxP95) return null;
+      const coalesceTo = Math.max(1, Math.round(stage.taskCount / 10));
+      const fix = stage.shuffleReadBytes > 0
+        ? `lower spark.sql.shuffle.partitions or .coalesce(${coalesceTo})`
+        : `.coalesce(${coalesceTo})`;
+      return {
+        type: 'tinyTask', stageId: stage.id, impactBand: 'info',
+        metric: 'taskDurationP50', value: Math.round(stage.taskDurationP50),
+        recommendation: `Many small tasks (${stage.taskCount}, P50 ${Math.round(stage.taskDurationP50)}ms): scheduler overhead may dominate. Try ${fix}.`,
+      };
+    },
+  },
+  {
+    // No docAnchor set (documented deviation, like autoscalingChurn above):
+    // the upstream `shuffle-works/spark-tuning-reference` docs repo has no
+    // section for this tool-specific "capture stopped early" signal.
+    type: 'incompleteRun', scope: 'app', order: 5, fixEffort: 'code', version: 1,
+    thresholds: {},
+    recommendation: 'This event log never recorded an ApplicationEnd event: the capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Findings and metrics elsewhere on this board reflect only what was captured up to that point, not the full run.',
+    detect(                                  ctx             )                 {
+      if (ctx.app.startTime == null || ctx.app.endTime != null) return null;
+      return {
+        type: 'incompleteRun', stageId: null, impactBand: 'warning',
+        metric: 'applicationEnd', value: 'missing',
+        recommendation: this.recommendation,
+      };
+    },
+  },
+  {
+    type: 'coldStart', scope: 'app', order: 90, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-cold-start',
+    thresholds: { gapSeconds: 30 },
+    detect(
+                                                   
+      ctx             ,
+    )                 {
+      const { app, stages } = ctx;
+      if (!app.startTime || stages.size === 0) return null;
+      let firstTaskLaunch = Infinity;
+      for (const stage of stages.values()) {
+        if (stage.submittedAt > 0 && stage.submittedAt < firstTaskLaunch) firstTaskLaunch = stage.submittedAt;
+      }
+      const gapSeconds = (firstTaskLaunch - app.startTime) / 1000;
+      if (gapSeconds <= this.thresholds.gapSeconds) return null;
+      const value = Math.round(gapSeconds);
+      return {
+        type: 'coldStart', stageId: null, impactBand: 'warning',
+        metric: 'startupGapSeconds', value,
+        recommendation: `Executor startup took ${value}s: consider pre-warming the cluster or using dynamic allocation.`,
+      };
+    },
+  },
+  {
+    type: 'utilization', scope: 'app', order: 100, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-utilization',
+    thresholds: { minUtil: 0.60 },
+    detect(
+                                                
+      ctx             ,
+    )                 {
+      const { app, executorsAdded, executorsRemoved } = ctx;
+      if (executorsAdded.length === 0 || !app.startTime || !app.endTime) return null;
+      const appDuration = app.endTime - app.startTime;
+      if (appDuration <= 0) return null;
+      const peakExecutors = executorsAdded.length;
+      let totalActiveMs = 0;
+      const removed = new Map                ();
+      for (const ev of executorsRemoved) removed.set(ev.executorId, ev.timestamp);
+      for (const ev of executorsAdded) {
+        const addedAt = Math.max(ev.timestamp, app.startTime);
+        const removedAt = removed.has(ev.executorId) ? removed.get(ev.executorId)  : app.endTime;
+        totalActiveMs += Math.max(0, removedAt - addedAt);
+      }
+      const utilization = (totalActiveMs / appDuration) / peakExecutors;
+      if (utilization >= this.thresholds.minUtil) return null;
+
+      // CPU-time-based utilization (sparkMeasure): metric only, no threshold.
+      // Total cores: prefer executor-added Total Cores (real), else config cores.
+      const totalCores = computeTotalCores(app, executorsAdded);
+      let cpuUtilizationPct                = null;
+      if (totalCores > 0) {
+        let cpuMs = 0;
+        // executorCpuTime is reported by Spark in nanoseconds.
+        for (const s of ctx.stages.values()) cpuMs += nsToMs(s.executorCpuTime ?? 0);
+        cpuUtilizationPct = Math.round((cpuMs / (appDuration * totalCores)) * 100);
+      }
+
+      const value = Math.round(utilization * 100);
+      return {
+        type: 'utilization', stageId: null, impactBand: 'info',
+        metric: 'avgUtilization', value,
+        utilizationFraction: utilization,
+        appDurationMs: appDuration,
+        totalCores,
+        cpuUtilizationPct,
+        recommendation: `Average executor utilization was only ${value}%: consider reducing cluster size or enabling dynamic allocation.`,
+      };
+    },
+  },
+  {
+    type: 'memoryUtilization', scope: 'app', order: 102, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-memory-utilization',
+    thresholds: {
+      idleCoreWarn: 0.50,          // dataflint WastedCoresAlertsReducer
+      bandTooSmall: 0.95,          // dataflint MemoryAlertsReducer: used/allocated
+      bandTooHigh: 0.70,           // below this => over-provisioned (cost signal)
+      wasteBufferMultiplier: 1.5,  // Dr. Elephant SparkMetricsAggregator: UNVERIFIED
+    },
+    detect(
+             
+                     
+                                                                                                         
+          
+        
+      ctx             ,
+    )            {
+      const { app, executorsAdded, runAggregates, stages } = ctx;
+      const out            = [];
+      // Nullish (not falsy) check: unlike the older coldStart/utilization
+      // detectors, a literal startTime:0 must not be treated as "missing".
+      if (app?.startTime == null || app?.endTime == null) return out;
+      const appDurationMs = app.endTime - app.startTime;
+      if (appDurationMs <= 0) return out;
+
+      // Total cores: prefer real Executor-Added Total Cores, else config.
+      const peakExecutors = executorsAdded.length;
+      const totalCores = computeTotalCores(app, executorsAdded);
+      // Hoisted above 1a (it is also 1b/1c's input) so the idle-cores finding can carry
+      // the allocated memory its MB-seconds impact estimate needs.
+      const allocatedMB = app.resources?.executor?.memoryMB ?? null;
+
+      // ── 1a idle-cores rate ────────────────────────────────────────────────
+      if (runAggregates && totalCores > 0) {
+        const capacityCoreMs = totalCores * appDurationMs;
+        const idleRate = capacityCoreMs > 0 ? 1 - (runAggregates.busyCoreMs / capacityCoreMs) : 0;
+        if (idleRate > this.thresholds.idleCoreWarn) {
+          const value = Math.round(idleRate * 100);
+          out.push({
+            type: 'memoryUtilization', variant: 'idleCores', stageId: null,
+            impactBand: 'warning', metric: 'idleCoreRate', value,
+            // Raw (unrounded) rate plus the sizing inputs, for the impact estimator's
+            // wasted-MB-seconds model: `value` above is a rounded percentage.
+            idleRateFraction: idleRate, allocatedMB, peakExecutors, appDurationMs,
+            recommendation: `${value}% of allocated core-time ran no task: reduce cluster size or enable dynamic allocation.`,
+          });
+        }
+      }
+
+      // ── 1b memory bands (executor only; driver half dropped since there is no driver metric) ─
+      // Peak heap per executor = max jvmHeapMemory across all stages' executorMetrics.
+      const peakHeapByExec = new Map                ();
+      for (const s of stages.values()) {
+        const em = s.executorMetrics;
+        if (!(em instanceof Map)) continue;
+        for (const [execId, m] of em) {
+          const heap = m?.jvmHeapMemory ?? 0;
+          if (heap > (peakHeapByExec.get(execId) ?? 0)) peakHeapByExec.set(execId, heap);
+        }
+      }
+      if (peakHeapByExec.size === 0) {
+        out.push({
+          type: 'memoryUtilization', variant: 'memoryBand', stageId: null,
+          impactBand: 'info', metric: 'memoryBand', dataUnavailable: true,
+          recommendation: 'Per-executor memory usage requires spark.eventLog.logStageExecutorMetrics=true: not enabled for this run.',
+        });
+      } else if (allocatedMB != null && allocatedMB > 0) {
+        const allocatedBytes = allocatedMB * 1024 * 1024;
+        for (const [execId, heap] of peakHeapByExec) {
+          const ratio = heap / allocatedBytes;
+          // The two bands are opposite signals, not two degrees of one: an explicit
+          // `rule` discriminator (same pattern as stageShape) lets consumers tell the
+          // OOM-risk case from the over-provisioning waste case without re-deriving
+          // the ratio against the thresholds.
+          if (ratio > this.thresholds.bandTooSmall) {
+            out.push({
+              type: 'memoryUtilization', variant: 'memoryBand', rule: 'heapNearCapacity',
+              stageId: null, executorId: execId,
+              impactBand: 'warning', metric: 'heapUsedRatio', value: Math.round(ratio * 100),
+              recommendation: `Executor ${execId} peaked at ${Math.round(ratio * 100)}% of allocated heap: memory may be too small; raise spark.executor.memory to avoid OOM/spill.`,
+            });
+          } else if (ratio < this.thresholds.bandTooHigh) {
+            out.push({
+              type: 'memoryUtilization', variant: 'memoryBand', rule: 'heapOverProvisioned',
+              stageId: null, executorId: execId,
+              impactBand: 'info', metric: 'heapUsedRatio', value: Math.round(ratio * 100),
+              // Absolute figures behind the rounded ratio, for the estimator's
+              // unused-memory-over-time model.
+              allocatedBytes, heap, appDurationMs,
+              recommendation: `Executor ${execId} used only ${Math.round(ratio * 100)}% of allocated heap: memory may be over-provisioned; consider reducing spark.executor.memory for cost savings.`,
+            });
+          }
+        }
+      }
+
+      // ── 1c Spark Memory Limit waste model (Dr. Elephant, UNVERIFIED buffer) ─
+      if (allocatedMB != null && peakExecutors > 0) {
+        const allocatedMBSeconds = peakExecutors * allocatedMB * (appDurationMs / 1000);
+        let usedRunTimeMs = 0;
+        for (const s of stages.values()) usedRunTimeMs += s.executorRunTime ?? 0;
+        const usedMBSeconds = allocatedMB * (usedRunTimeMs / 1000);
+        const wastedMBSeconds = allocatedMBSeconds - usedMBSeconds;
+        if (wastedMBSeconds > this.thresholds.wasteBufferMultiplier * usedMBSeconds) {
+          const value = Math.round(wastedMBSeconds);
+          out.push({
+            type: 'memoryUtilization', variant: 'wasteModel', stageId: null,
+            impactBand: 'info', metric: 'wastedMBSeconds', value,
+            confidence: 'low',
+            validationRequired: 'Memory-waste estimate uses allocated-vs-used memory-time and an unverified 1.5x buffer ported from Dr. Elephant: confirm against the Spark UI before acting.',
+            recommendation: `Allocated executor memory sat largely idle over the run (~${value.toLocaleString('en-US')} MB-seconds wasted): review spark.executor.memory and executor count.`,
+          });
+        }
+      }
+
+      return out;
+    },
+  },
+  {
+    // Per-RDD cache-utilization proxies (this repo's own design: Spark
+    // event logs carry no block-access/read-count events, so a literal
+    // cache hit rate isn't derivable; see the design doc for GitHub issue
+    // #85). Two independent, per-RDD tiered checks over `ctx.app.rddInfo`
+    // storage snapshots: partial caching (numCachedPartitions < numPartitions)
+    // and disk spillover (diskSize share of a MEMORY_AND_DISK*-requesting
+    // RDD's cached footprint). An RDD can produce both findings in one pass.
+    type: 'cacheUtilization', scope: 'app', order: 103, fixEffort: 'code', version: 1,
+    docAnchor: '#memory-model',
+    thresholds: {
+      cachedRatioWarn: 0.50, cachedRatioInfo: 0.90,
+      diskRatioWarn: 0.40, diskRatioInfo: 0.15,
+    },
+    detect(
+             
+                     
+                                                                                                         
+          
+        
+      ctx             ,
+    )                   {
+      const rddInfo = ctx.app?.rddInfo;
+      if (!(rddInfo instanceof Map)) return null;
+      const out            = [];
+      for (const rdd of rddInfo.values()) {
+        const sl = rdd.storageLevel ?? {};
+        if (!(sl.useMemory || sl.useDisk)) continue;
+        if (!((rdd.numCachedPartitions ?? 0) > 0)) continue;
+
+        if ((rdd.numPartitions ?? 0) > 0) {
+          const cachedRatio = rdd.numCachedPartitions / rdd.numPartitions;
+          if (cachedRatio < this.thresholds.cachedRatioWarn) out.push(partialCacheFinding(rdd, cachedRatio, 'warning'));
+          else if (cachedRatio < this.thresholds.cachedRatioInfo) out.push(partialCacheFinding(rdd, cachedRatio, 'info'));
+        }
+
+        if (sl.useMemory && sl.useDisk) {
+          const total = (rdd.memorySize ?? 0) + (rdd.diskSize ?? 0);
+          if (total > 0) {
+            const diskRatio = (rdd.diskSize ?? 0) / total;
+            if (diskRatio > this.thresholds.diskRatioWarn) out.push(diskSpilloverFinding(rdd, diskRatio, 'warning'));
+            else if (diskRatio > this.thresholds.diskRatioInfo) out.push(diskSpilloverFinding(rdd, diskRatio, 'info'));
+          }
+        }
+      }
+      return out;
+    },
+  },
+  {
+    // Non-local task ratio across stage.localityStats (RACK_LOCAL + ANY vs.
+    // all tasks), the other half of dataflint's "Wasted Cores Ratio" alert;
+    // the idle-core half already lives in memoryUtilization's idleCores
+    // variant above. NO_PREF stays in the denominator only: it's what
+    // shuffle-read stages legitimately report with no locality problem.
+    type: 'coreLocality', scope: 'app', order: 103, fixEffort: 'config', version: 1,
+    docAnchor: '#bottleneck-utilization',
+    thresholds: { minTasks: 50, warnRatio: 0.15, critRatio: 0.35 },
+    detect(
+                                                                                       
+      ctx             ,
+    )                 {
+      const { totalTasks, nonLocalTasks, ratio } = computeCoreLocalityRatio([...ctx.stages.values()]);
+      if (totalTasks == null || totalTasks < this.thresholds.minTasks) return null;
+      // computeCoreLocalityRatio (src/core-locality-ratio.ts) only ever returns
+      // `ratio: null` together with `totalTasks: null` (its EMPTY sentinel sets
+      // both at once); the `totalTasks == null` guard above already rules that
+      // out, so `ratio` is guaranteed non-null here even though the function's
+      // declared return type keeps the two nullable independently.
+      if (ratio  < this.thresholds.warnRatio) return null;
+
+      const value = Math.round(ratio  * 100);
+      return {
+        type: 'coreLocality', stageId: null,
+        impactBand: ratio  >= this.thresholds.critRatio ? 'critical' : 'warning',
+        metric: 'nonLocalRatio', value,
+        // Raw count behind the ratio, for the impact estimator's network-fetch-penalty
+        // figure. Non-null whenever totalTasks is (both come from the same EMPTY sentinel).
+        nonLocalTaskCount: nonLocalTasks ,
+        confidence: 'low',
+        validationRequired: 'The 15%/35% non-local-ratio thresholds (and the 50-task minimum) are unvalidated design-spike values: no external tool publishes an equivalent metric to calibrate against. Confirm against known-good/known-bad real logs before trusting the impact-band split.',
+        recommendation: `${value}% of tasks (${nonLocalTasks }) ran without process- or node-local data placement: check spark.locality.wait settings and executor/data colocation.`,
+      };
+    },
+  },
+  {
+    // Short-lived executors: an executor stood up and torn down before it
+    // could do useful work: wasteful re-provisioning, not normal scale-down.
+    // Reuses the same executorsAdded/executorsRemoved matching logic as
+    // `utilization` above (no new data extraction), but measures lifetime
+    // against a threshold instead of aggregate active-time.
+    type: 'autoscalingChurn', scope: 'app', order: 103, fixEffort: 'config', version: 1,
+    confidence: 'low',
+    thresholds: { shortLivedMs: 120_000, warningPct: 0.30, criticalPct: 0.60, minExecutors: 5 },
+    detect(
+             
+                                                                                                            
+                           
+        
+      ctx             ,
+    )                 {
+      const { app, executorsAdded, executorsRemoved } = ctx;
+      if (executorsAdded.length === 0 || app.endTime == null) return null;
+      if (executorsAdded.length < this.thresholds.minExecutors) return null;
+
+      const removedAt = new Map                ();
+      for (const ev of executorsRemoved) removedAt.set(ev.executorId, ev.timestamp);
+
+      let shortLivedCount = 0;
+      for (const ev of executorsAdded) {
+        const endedAt = removedAt.has(ev.executorId) ? removedAt.get(ev.executorId)  : app.endTime;
+        const lifetime = endedAt - ev.timestamp;
+        if (lifetime < this.thresholds.shortLivedMs) shortLivedCount++;
+      }
+
+      const shortLivedPct = shortLivedCount / executorsAdded.length;
+      const impactBand = shortLivedPct > this.thresholds.criticalPct ? 'critical'
+                        : shortLivedPct > this.thresholds.warningPct ? 'warning' : null;
+      if (!impactBand) return null;
+
+      const pct = Math.round(shortLivedPct * 100);
+      return {
+        type: 'autoscalingChurn', stageId: null, impactBand,
+        metric: 'shortLivedExecutorPct', value: pct,
+        // Raw count behind the percentage, for the impact estimator's startup-overhead figure.
+        shortLivedExecutorCount: shortLivedCount,
+        confidence: this.confidence,
+        recommendation: `${pct}% of executors ran for under 2 minutes before being removed. This looks like wasteful re-provisioning rather than normal scale-down; consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.`,
+      };
+    },
+  },
+  {
+    // Cross-execution relation reuse (repurposed from the old RDD-lineage
+    // heuristic, which surfaced only internal query-engine RDDs on DataFrame/
+    // SQL workloads; see docs/adr/0009-caching-opportunity-relation-reuse.md).
+    // Flags an input relation scanned by two or more SQL executions in one run.
+    type: 'cachingOpportunity', scope: 'app', order: 105, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-utilization',
+    thresholds: { minExecutions: 2 },
+    detect(                                                 ctx             )                   {
+      const sql = ctx.sql;
+      if (!(sql instanceof Map) || sql.size === 0) return null;
+
+                             
+                                         
+                                                                       
+       
+                                     
+                                                   
+                                                                                  
+       
+                                
+                                                          
+                                                                       
+                                                                         
+       
+                                                                                            
+
+      // relationId -> { format, relation, executionIds:Set, executionBytes: Map<execId, bytes> }
+      const byRelation = new Map                     ();
+      for (const exec of sql.values()) {
+        if (!exec.planTree) continue;
+        // Dedupe relations within one execution (self-joins count once), summing
+        // this execution's read bytes per relation across its scan nodes.
+        const perExec = new Map                ();
+        walkPlanTree(exec.planTree, (node) => {
+          const rid = scanRelationId(node.name ?? '', node.detail ?? '');
+          if (!rid) return;
+          const bytesMetric = (node.metrics ?? []).find(m => m.name === FILES_READ_BYTES);
+          perExec.set(rid, (perExec.get(rid) ?? 0) + (bytesMetric ? bytesMetric.value : 0));
+        });
+        for (const [rid, bytes] of perExec) {
+          let agg = byRelation.get(rid);
+          if (!agg) {
+            const colon = rid.indexOf(':');
+            agg = { format: rid.slice(0, colon), relation: rid.slice(colon + 1), executionIds: new Set(), executionBytes: new Map() };
+            byRelation.set(rid, agg);
+          }
+          agg.executionIds.add(exec.id);
+          agg.executionBytes.set(exec.id, (agg.executionBytes.get(exec.id) ?? 0) + bytes);
+        }
+      }
+
+      // fingerprint -> { operator, exampleNode, executionIds:Set, executionBytes:Map<execId,bytes>, ancestorFingerprints:Set<fingerprint>, leafRelationRids:Set<rid> }
+      const byComposite = new Map                        ();
+      for (const exec of sql.values()) {
+        if (!exec.planTree) continue;
+        const candidates = findCompositeCandidates(exec.planTree);
+        const fingerprintByNode = new Map                  (candidates.map((c)                     => [c.node, c.fingerprint]));
+
+        // Dedupe identical fingerprints within this execution (repeated
+        // identical composite counts once, mirroring the leaf perExec dedupe).
+        const perExecComposite = new Map                             ();
+        for (const c of candidates) {
+          let agg = perExecComposite.get(c.fingerprint);
+          if (!agg) {
+            agg = {
+              operator: c.operator, node: c.node,
+              leafRelationBytes: new Map(),
+              ancestorFingerprints: new Set(
+                c.ancestorNodes.map(n => fingerprintByNode.get(n)).filter((fp)               => Boolean(fp)),
+              ),
+            };
+            perExecComposite.set(c.fingerprint, agg);
+          }
+          for (const [rid, bytes] of c.leafRelationBytes) {
+            agg.leafRelationBytes.set(rid, (agg.leafRelationBytes.get(rid) ?? 0) + bytes);
+          }
+        }
+
+        for (const [fingerprint, agg] of perExecComposite) {
+          let cAgg = byComposite.get(fingerprint);
+          if (!cAgg) {
+            cAgg = {
+              operator: agg.operator, exampleNode: agg.node,
+              executionIds: new Set(), executionBytes: new Map(),
+              ancestorFingerprints: new Set(), leafRelationRids: new Set(),
+            };
+            byComposite.set(fingerprint, cAgg);
+          }
+          cAgg.executionIds.add(exec.id);
+          const execBytes = [...agg.leafRelationBytes.values()].reduce((sum, b) => sum + b, 0);
+          cAgg.executionBytes.set(exec.id, (cAgg.executionBytes.get(exec.id) ?? 0) + execBytes);
+          for (const af of agg.ancestorFingerprints) cAgg.ancestorFingerprints.add(af);
+          for (const rid of agg.leafRelationBytes.keys()) cAgg.leafRelationRids.add(rid);
+        }
+      }
+
+      // Qualifying = has enough distinct executions on its own. Nested-dedupe:
+      // if a qualifying composite has a qualifying ANCESTOR composite, it is
+      // subsumed: fully (equal execution sets) or partially (residual).
+      const isQualifying = (fp        )          =>
+        byComposite.has(fp) && byComposite.get(fp) .executionIds.size >= this.thresholds.minExecutions;
+      // fingerprint -> { finalExecutionIds:Set, suppressed:boolean }
+      const compositeResolutions = new Map                             ();
+      for (const [fingerprint, agg] of byComposite) {
+        if (!isQualifying(fingerprint)) { compositeResolutions.set(fingerprint, { finalExecutionIds: agg.executionIds, suppressed: true }); continue; }
+        const qualifyingAncestors = [...agg.ancestorFingerprints].filter(isQualifying).map(fp => byComposite.get(fp) );
+        if (qualifyingAncestors.length === 0) { compositeResolutions.set(fingerprint, { finalExecutionIds: agg.executionIds, suppressed: false }); continue; }
+        const coveredByAncestors = new Set(qualifyingAncestors.flatMap(outer => [...outer.executionIds]));
+        const residual = new Set([...agg.executionIds].filter(id => !coveredByAncestors.has(id)));
+        if (residual.size === 0) compositeResolutions.set(fingerprint, { finalExecutionIds: residual, suppressed: true });
+        else if (residual.size < this.thresholds.minExecutions) compositeResolutions.set(fingerprint, { finalExecutionIds: residual, suppressed: true });
+        else compositeResolutions.set(fingerprint, { finalExecutionIds: residual, suppressed: false });
+      }
+
+      const relationDisplayName = (rid        )         => rid.slice(rid.indexOf(':') + 1);
+      const compositeVerb                                             = { join: ['joined', 'join'], union: ['unioned', 'union'] };
+
+      const out            = [];
+      // rid -> Set<execId> covered by an emitted composite (for leaf suppression, Task 6)
+      const coveredExecutionsByRid = new Map                     ();
+      for (const [fingerprint, agg] of byComposite) {
+        const resolution = compositeResolutions.get(fingerprint) ;
+        if (resolution.suppressed) continue;
+        const finalExecutionIds = [...resolution.finalExecutionIds].sort((a, b) => a - b);
+        const totalReadBytes = finalExecutionIds.reduce((sum, id) => sum + (agg.executionBytes.get(id) ?? 0), 0);
+        const rids = [...agg.leafRelationRids].sort();
+        if (rids.length === 0) continue;
+        const relations = rids.map(rid => {
+          const colon = rid.indexOf(':');
+          return { relation: rid.slice(colon + 1), format: rid.slice(0, colon) };
+        });
+        const [verb, connector] = compositeVerb[agg.operator];
+        const relationDisplay = rids.map(relationDisplayName).join(` ${connector} `);
+        const value = finalExecutionIds.length;
+        const recommendation = totalReadBytes >= 128 * MB
+          ? `${verb[0].toUpperCase()}${verb.slice(1)} result read by ${value} queries (~${formatBytes(totalReadBytes)}). Cache/persist the ${verb} DataFrame so it is computed once.`
+          : `${verb[0].toUpperCase()}${verb.slice(1)} result read by ${value} queries. Cache the ${verb} DataFrame, or reconsider whether it needs to be recomputed each time.`;
+
+        out.push({
+          type: 'cachingOpportunity', variant: 'composite', stageId: null, impactBand: 'info',
+          metric: 'executionReuse', value,
+          format: 'derived', relations, operator: agg.operator, relation: relationDisplay,
+          executionIds: finalExecutionIds, totalReadBytes,
+          confidence: 'low',
+          validationRequired:
+            'Composite reuse is inferred from a structural plan-shape match (operator + normalized ' +
+            'join/filter condition + child shapes) across SQL executions; confirm these executions ' +
+            'truly compute the same join/union before caching.',
+          recommendation,
+        });
+
+        for (const rid of rids) {
+          if (!coveredExecutionsByRid.has(rid)) coveredExecutionsByRid.set(rid, new Set());
+          for (const id of finalExecutionIds) coveredExecutionsByRid.get(rid) .add(id);
+        }
+      }
+
+      for (const [rid, agg] of byRelation) {
+        const covered = coveredExecutionsByRid.get(rid);
+        const residualExecutionIds = covered
+          ? [...agg.executionIds].filter(id => !covered.has(id))
+          : [...agg.executionIds];
+        if (residualExecutionIds.length < this.thresholds.minExecutions) continue;
+        const value = residualExecutionIds.length;
+        const totalReadBytes = residualExecutionIds.reduce((sum, id) => sum + (agg.executionBytes.get(id) ?? 0), 0);
+        const recommendation = totalReadBytes >= 128 * MB
+          ? `Read by ${value} queries (~${formatBytes(totalReadBytes)}). Cache/persist the shared DataFrame so it is scanned once.`
+          : `Read by ${value} queries. Cache the shared DataFrame, or broadcast it if it is a small join lookup.`;
+        out.push({
+          type: 'cachingOpportunity', stageId: null, impactBand: 'info',
+          metric: 'executionReuse', value,
+          relation: agg.relation, format: agg.format,
+          executionIds: residualExecutionIds.sort((a, b) => a - b),
+          totalReadBytes,
+          confidence: 'low',
+          validationRequired:
+            'Relation-reuse is inferred from the pre-AQE plan scan identity across SQL ' +
+            'executions; confirm the reads are the same data and cacheable within one ' +
+            'session before acting.',
+          recommendation,
+        });
+      }
+      return out;
+    },
+  },
+  {
+    type: 'jobFailureRate', scope: 'app', order: 110, fixEffort: 'code', version: 1,
+    docAnchor: '#bottleneck-job-failure-rate',
+    thresholds: { infoRate: 0.10, warnRate: 0.30, critRate: 0.50 },
+    detect(
+                                                                                     
+      ctx             ,
+    )                 {
+      const { jobs, stages } = ctx;
+      const all = jobs ? [...jobs.values()] : [];
+      const completed = all.filter(j => j.result != null);
+      if (completed.length === 0) return null;
+      const failedJobList = completed.filter(j => j.succeeded === false);
+      const failedJobs = failedJobList.length;
+      const rate = failedJobs / completed.length;
+      if (rate < this.thresholds.infoRate) return null;
+      let totalTasks = 0, failedTasks = 0;
+      for (const s of stages.values()) { totalTasks += s.taskCount ?? 0; failedTasks += s.failedTasks ?? 0; }
+      const taskFailureRate = totalTasks > 0 ? failedTasks / totalTasks : 0;
+      // Average wall-clock duration of the failed jobs, for the impact estimator's
+      // cost-only "core-hours burned on work that was thrown away" figure. Jobs
+      // missing either timestamp are excluded rather than counted as zero-length;
+      // with no timed failed job at all the average is 0 (never NaN).
+      const timedFailedJobs = failedJobList.filter(j => j.submissionTime != null && j.completionTime != null);
+      const avgJobDurationMs = timedFailedJobs.length > 0
+        ? timedFailedJobs.reduce((s, j) => s + (j.completionTime  - j.submissionTime ), 0) / timedFailedJobs.length
+        : 0;
+      const totalJobs = completed.length;
+      return {
+        type: 'jobFailureRate', stageId: null,
+        impactBand: rate >= this.thresholds.critRate ? 'critical' : rate >= this.thresholds.warnRate ? 'warning' : 'info',
+        metric: 'jobFailureRate', value: Math.round(rate * 1000) / 10,
+        failedJobs, totalJobs, failedTasks, totalTasks, avgJobDurationMs,
+        taskFailureRate: Math.round(taskFailureRate * 1000) / 10,
+        recommendation: `${failedJobs} of ${totalJobs} jobs never recovered: inspect the driver log for the failed job(s) and the stage failures that triggered them.`,
+      };
+    },
+  },
+  // ── Config-sanity entries (scope:'config', inScorecard:false) ────────────────
+  {
+    type: 'configAudit', scope: 'config', order: 120, fixEffort: 'config', version: 1, inScorecard: false,
+    docAnchor: '#config-shuffle-service', thresholds: {}, property: 'spark.shuffle.service.enabled',
+    detect(ctx                      )                 {
+      const res = ctx.app?.resources ?? null;
+      if (res?.dynamicAllocationEnabled === true && res?.shuffleServiceEnabled === false) {
+        return {
+          type: 'configAudit', property: 'spark.shuffle.service.enabled',
+          impactBand: 'warning', metric: 'config', value: 'false',
+          recommendation: 'Dynamic allocation is on but the external shuffle service is off: set spark.shuffle.service.enabled=true so shuffle data survives executor removal.',
+        };
+      }
+      return null;
+    },
+  },
+  {
+    type: 'configAudit', scope: 'config', order: 121, fixEffort: 'config', version: 1, inScorecard: false,
+    docAnchor: '#config-autoscale-bounds', thresholds: {}, property: 'spark.dynamicAllocation.maxExecutors',
+    detect(ctx                      )                 {
+      const app = ctx.app; const config = app?.config ?? {}; const res = app?.resources ?? null;
+      if (res?.dynamicAllocationEnabled !== true) return null;
+      const minN = config['spark.dynamicAllocation.minExecutors'] != null ? parseInt(config['spark.dynamicAllocation.minExecutors'], 10) : null;
+      const maxN = config['spark.dynamicAllocation.maxExecutors'] != null ? parseInt(config['spark.dynamicAllocation.maxExecutors'], 10) : null;
+      if (minN != null && maxN != null && minN > maxN) {
+        return {
+          type: 'configAudit', property: 'spark.dynamicAllocation.minExecutors',
+          impactBand: 'critical', metric: 'config', value: `${minN} > ${maxN}`,
+          recommendation: `Autoscaling bounds are inverted: spark.dynamicAllocation.minExecutors (${minN}) exceeds maxExecutors (${maxN}). Set min ≤ max.`,
+        };
+      }
+      if (maxN == null) {
+        return {
+          type: 'configAudit', property: 'spark.dynamicAllocation.maxExecutors',
+          impactBand: 'info', metric: 'config', value: '(unset)',
+          recommendation: 'Dynamic allocation is on with no upper bound: set spark.dynamicAllocation.maxExecutors to cap cluster growth.',
+        };
+      }
+      return null;
+    },
+  },
+  {
+    type: 'configAudit', scope: 'config', order: 122, fixEffort: 'config', version: 1, inScorecard: false,
+    docAnchor: '#config-serializer', thresholds: {}, property: 'spark.serializer',
+    detect(ctx                      )                 {
+      const app = ctx.app; const config = app?.config ?? {}; const res = app?.resources ?? null;
+      if (Object.keys(config).length === 0) return null;
+      const ser = res?.serializer ?? config['spark.serializer'] ?? null;
+      const isKryo = typeof ser === 'string' && /kryo/i.test(ser);
+      if (isKryo) return null;
+      return {
+        type: 'configAudit', property: 'spark.serializer',
+        impactBand: 'info', metric: 'config', value: ser ?? '(default JavaSerializer)',
+        recommendation: `Current serializer is ${ser ?? 'the default JavaSerializer'}: consider spark.serializer=org.apache.spark.serializer.KryoSerializer for faster, smaller buffers.`,
+      };
+    },
+  },
+  {
+    type: 'configAudit', scope: 'config', order: 123, fixEffort: 'config', version: 1, inScorecard: false,
+    docAnchor: '#config-memory-overhead', thresholds: { floorMB: 384, floorPct: 0.1 }, property: 'spark.executor.memoryOverhead',
+    detect(
+                                                                  
+      ctx                      ,
+    )                 {
+      const res = ctx.app?.resources ?? null;
+      const memMB = res?.executor?.memoryMB ?? null;
+      const ovMB = res?.executor?.memoryOverheadMB ?? null;
+      if (memMB == null || ovMB == null) return null;
+      const floor = Math.max(this.thresholds.floorMB, Math.round(memMB * this.thresholds.floorPct));
+      if (ovMB >= floor) return null;
+      return {
+        type: 'configAudit', property: 'spark.executor.memoryOverhead',
+        impactBand: 'info', metric: 'config', value: `${ovMB} MiB`,
+        recommendation: `Executor memoryOverhead (${ovMB} MiB) is below Spark's default floor of ${floor} MiB (max of 384 MiB or 10% of executor memory): raise it to avoid off-heap OOM-kills.`,
+      };
+    },
+  },
+  // ── Plan-metric entries (scope:'sql') ────────────────────────────────────
+  {
+    type: 'duplicatePlanSubtree', scope: 'sql', order: 130, fixEffort: 'code', version: 2,
+    docAnchor: '#bottleneck-duplicate-plan-subtree',
+    thresholds: { minSubtreeSize: 3, minOccurrences: 2 },
+    detect(
+                                                                               
+      sqlExec                 ,
+      ctx             ,
+    )                   {
+      if (!sqlExec.planTree) return null;
+      const groups = findDuplicateSubtrees(sqlExec.planTree, this.thresholds);
+      if (groups.length === 0) return null;
+      const fallbackStageIds = stageIdsForSqlExec(sqlExec.id, ctx.stages);
+      return groups.map((g) => {
+        const nodes             = [];
+        for (const n of g.nodes) walkPlanTree(n, (node) => nodes.push(node));
+        const stageIds = unionStageIds(nodes, fallbackStageIds);
+        const touching = g.sampleRelation ? ` (touching ${g.sampleRelation})` : '';
+        return {
+          type: 'duplicatePlanSubtree', executionId: sqlExec.id, stageIds,
+          // Fixed fallback: overwritten by deriveImpactBand whenever this finding
+          // gets a real wallClock estimate, which is the common case. Only
+          // surfaces on the rare miss (stage excluded from the occupancy sweep).
+          impactBand: 'warning', metric: 'subtreeOccurrences', value: g.occurrences,
+          rootName: g.rootName, subtreeSize: g.subtreeSize, sampleRelation: g.sampleRelation,
+          groupIndex: g.groupIndex, confidence: 'medium',
+          validationRequired: 'Duplicate-subtree matching compares operator names and metric names only, not literal values or expr IDs: confirm the repeated work is real in the Spark SQL plan tab before acting.',
+          recommendation: g.isExchangeRoot
+            ? `A ${g.subtreeSize}-node subtree rooted at ${pathBasename(g.rootName)} repeats ${g.occurrences}x in this plan${touching}: this looks like a possible missed exchange reuse; check whether the same shuffle could be computed once and reused.`
+            : `A ${g.subtreeSize}-node subtree rooted at ${pathBasename(g.rootName)} repeats ${g.occurrences}x in this plan${touching}: consider caching/persisting the shared computation or check for a duplicated query branch.`,
+        };
+      });
+    },
+  },
+  {
+    type: 'smallFiles', scope: 'sql', order: 131, fixEffort: 'config', version: 2,
+    docAnchor: '#bottleneck-small-files',
+    thresholds: { minFiles: 100, maxAvgFileSizeMB: 3 },
+    detect(
+                                                                           
+      sqlExec                 ,
+      ctx             ,
+    )                   {
+      if (!sqlExec.planTree) return null;
+      const { minFiles, maxAvgFileSizeMB } = this.thresholds;
+                                                                                                                                     
+      const hits                  = [];
+      walkPlanTree(sqlExec.planTree, (node) => {
+        const metrics = node.metrics ?? [];
+        const byName = (name        ) => metrics.find(m => m.name === name);
+        const checkSide = (
+          countMetric                                             ,
+          bytesMetric                                             ,
+          direction                  ,
+        ) => {
+          if (!countMetric || !bytesMetric || !(countMetric.value > minFiles)) return;
+          const avgBytes = bytesMetric.value / countMetric.value;
+          if (!(avgBytes < maxAvgFileSizeMB * MB)) return;
+          hits.push({ direction, fileCount: countMetric.value, avgBytes, nodeName: node.name, node });
+        };
+        checkSide(byName(FILES_READ_COUNT), byName(FILES_READ_BYTES), 'read');
+        checkSide(byName(FILES_WRITTEN_COUNT), byName(FILES_WRITTEN_BYTES), 'write');
+      });
+      if (hits.length === 0) return null;
+      const fallbackStageIds = stageIdsForSqlExec(sqlExec.id, ctx.stages);
+      return hits.map(h => {
+        const stageIds = unionStageIds([h.node], fallbackStageIds);
+        return {
+          type: 'smallFiles', executionId: sqlExec.id, stageIds,
+          impactBand: 'warning',
+          metric: 'avgFileSizeBytes', value: Math.round(h.avgBytes),
+          fileCount: h.fileCount, direction: h.direction, nodeName: h.nodeName,
+          recommendation: h.direction === 'read'
+            ? `${h.fileCount} small files were read at ${pathBasename(h.nodeName)}: consider compacting the upstream output so fewer, larger files are produced.`
+            : `${h.fileCount} small files were written at ${pathBasename(h.nodeName)}: repartition or coalesce before writing to raise the average file size.`,
+        };
+      });
+    },
+  },
+  {
+    // Entry-level type is an identifier only; it never appears on an
+    // emitted finding. Findings carry their own type ('underBroadcast' or
+    // 'overBroadcast') since one shared plan-walk covers both opposite-
+    // direction rules (dataflint JoinToBroadcastAlert / BroadcastTooLargeAlert).
+    type: 'broadcastSizing', scope: 'sql', order: 132, fixEffort: 'config', version: 2,
+    docAnchor: '#bottleneck-broadcast-sizing',
+    thresholds: {
+      broadcastTiers: [10 * MB, 100 * MB, GB, 5 * GB],
+      comparisonTiers: [10 * GB, 300 * GB, TB],
+      overBroadcastBytes: GB,
+    },
+    detect(
+             
+                                                                                                        
+        
+      sqlExec                 ,
+      ctx             ,
+    )                   {
+      if (!sqlExec.planTree) return null;
+      const { broadcastTiers, comparisonTiers, overBroadcastBytes } = this.thresholds;
+      const fallbackStageIds = stageIdsForSqlExec(sqlExec.id, ctx.stages);
+      const out            = [];
+      walkPlanTree(sqlExec.planTree, (node) => {
+        if (node.name === 'SortMergeJoin' && (node.children ?? []).length === 2) {
+          const [childA, childB] = node.children;
+          const a = sumBoundarySize(childA);
+          const b = sumBoundarySize(childB);
+          const smaller = Math.min(a, b);
+          const larger = Math.max(a, b);
+          if (smaller > 0) {
+            const fires = smaller < broadcastTiers[0]
+              || (smaller < broadcastTiers[1] && larger > comparisonTiers[0])
+              || (smaller < broadcastTiers[2] && larger > comparisonTiers[1])
+              || (smaller < broadcastTiers[3] && larger > comparisonTiers[2]);
+            if (fires) {
+              const contributors = [...boundarySizeContributors(childA), ...boundarySizeContributors(childB)];
+              out.push({
+                type: 'underBroadcast', executionId: sqlExec.id, stageIds: unionStageIds(contributors, fallbackStageIds),
+                impactBand: 'info', metric: 'smallerSideBytes', value: smaller,
+                largerSideBytes: larger,
+                recommendation: `The smaller input to this Sort Merge Join (${formatBytes(smaller)}) is well under the broadcast threshold relative to the larger side (${formatBytes(larger)}): this could have been a broadcast join. Consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.`,
+              });
+            }
+          }
+        }
+        if (node.name === 'BroadcastExchange') {
+          const m = (node.metrics ?? []).find(x => x.name === 'data size');
+          if (m && m.value > overBroadcastBytes) {
+            // The BroadcastExchange node's own metrics are computed on the
+            // driver and never appear on any TaskEnd (node.stageIds is
+            // always empty in real data); its child carries the
+            // executor-side metrics, so only the child is unioned in.
+            const child = (node.children ?? [])[0];
+            out.push({
+              type: 'overBroadcast', executionId: sqlExec.id,
+              stageIds: unionStageIds(child ? [child] : [], fallbackStageIds),
+              impactBand: 'warning', metric: 'broadcastBytes', value: m.value,
+              recommendation: `This broadcast (${formatBytes(m.value)}) exceeds the 1 GB threshold: check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.`,
+            });
+          }
+        }
+      });
+      return out.length ? out : null;
+    },
+  },
+];