sparkforensics-server 0.5.0 → 0.5.2

package/vendor-core/event-handlers.js

@@ -18,6 +18,7 @@ import {
                   
                      
 } from './event-schemas.js';
+import { DELTA_COMMANDS } from './delta-commands.js';
 import { assertNever } from './assert-never.js';
 import { finalizeStage } from './stage-quantiles.js';
 import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
@@ -198,6 +199,9 @@ const MAX_TASK_SAMPLES = 20;
                     
                          
                      
+                           
+                                                                                  
+                            
                                                                                          
                                       
                                                                                                   
@@ -279,6 +283,10 @@ export function buildChunkDecoder() {
   // line again after every slice scanned about 10 GB on a 100 MB line.
   let pendingIsSqlEvent = false;
   let keyTail = '';
+  // Where the pending line's plan-description value starts, once its key is found but too little
+  // of the value has arrived to tell whether it is a small Delta command description (see
+  // descriptionDisposition); -1 otherwise.
+  let undecidedValueStart = -1;
   let skippingPlanDescription = false;
   // Length of the backslash run the skipped bytes ended with, which escapes a quote at the start
   // of the next chunk when odd.
@@ -312,11 +320,24 @@ export function buildChunkDecoder() {
       keyTail = '';
       appended = pending; // nothing of it was searched while it was shorter than the prefix
     }
-    const keyAt = findPlanKey(appended);
-    if (keyAt === -1) return; // the key may still arrive in a later chunk
+    let valueStart = undecidedValueStart;
+    if (valueStart === -1) {
+      const keyAt = findPlanKey(appended);
+      if (keyAt === -1) return; // the key may still arrive in a later chunk
+      valueStart = keyAt + PLAN_DESCRIPTION_KEY.length;
+    }
+    if (closingQuoteIndex(pending, valueStart) !== -1) { // complete: stripPlanDescription handles it
+      undecidedValueStart = -1;
+      pendingSettled = true;
+      return;
+    }
+    // A small kept description is not skipped: the line completes in a later chunk and
+    // stripPlanDescription keeps its arguments line.
+    const disposition = descriptionDisposition(pending, valueStart);
+    if (disposition === 'wait') { undecidedValueStart = valueStart; return; }
+    undecidedValueStart = -1;
     pendingSettled = true;
-    const valueStart = keyAt + PLAN_DESCRIPTION_KEY.length;
-    if (closingQuoteIndex(pending, valueStart) !== -1) return; // complete: stripPlanDescription empties it
+    if (disposition === 'keep') return;
     // Only a chunk whose last byte is a backslash carries a run over; any other last byte (such
     // as part of a split multibyte char) ends it.
     let run = 0;
@@ -378,6 +399,7 @@ export function buildChunkDecoder() {
       pending = pendingHead = appended = start < text.length ? text.substring(start) : '';
       pendingSettled = false;
       pendingIsSqlEvent = false;
+      undecidedValueStart = -1;
     }
     if (!pendingSettled && pending !== '') settlePending(buffer[buffer.length - 1], appended);
   }
@@ -1109,6 +1131,10 @@ export function startSqlExecution(event
     sparkPlanInfo,
     hadAdaptiveUpdate: false,
   };
+  if (event.rootExecutionId !== undefined) exec.rootExecutionId = event.rootExecutionId;
+  if (event.physicalPlanDescription && KEPT_ARGUMENTS.test(event.physicalPlanDescription)) {
+    exec.commandArguments = event.physicalPlanDescription;
+  }
   state.sqlExecutions.set(exec.id, exec);
   // A restarted execution carries a new plan: its next end must resolve it again.
   state.resolvedPlanExecutions.delete(exec.id);
@@ -1295,13 +1321,32 @@ const KNOWN_EVENT_TYPES                      = new Set(
 );
 
 // SQLExecutionStart and SQLAdaptiveExecutionUpdate carry `physicalPlanDescription`, Spark's text
-// rendering of the plan. Nothing reads it (the plan tree comes from sparkPlanInfo), yet on a real
-// 3.5 GB log it was 72% of the AQE-update bytes, which were themselves 73% of the log. Cutting its
+// rendering of the plan. The plan tree comes from sparkPlanInfo, yet on a real 3.5 GB log the
+// description was 72% of the AQE-update bytes, which were themselves 73% of the log. Cutting its
 // string value out before JSON.parse halves the parse cost of those lines.
+//
+// One exception: a Delta write command's node prints only its bare name in sparkPlanInfo, and its
+// target table appears only on its `Arguments:` line of the description. When the description is
+// that of one of KEPT_ARGUMENTS_COMMANDS (checked on its first bytes, so no other line pays for it),
+// the value is cut down to that one line, never kept whole. A value that spans decoder chunks is
+// still dropped unread (buildChunkDecoder), so the write target then falls back to its child paths.
 const SQL_UI_EVENT_PREFIX = '{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQL';
 const PLAN_DESCRIPTION_KEY = '"physicalPlanDescription":"';
 
-const QUOTE = 0x22, BACKSLASH = 0x5c, NEWLINE = 0x0a;
+const QUOTE = 0x22, BACKSLASH = 0x5c, NEWLINE = 0x0a, LOWER_N = 0x6e;
+
+// Delta commands whose root node carries no target in sparkPlanInfo. The description of a
+// command execution starts `== Physical Plan ==\nExecute <Command> (1)` (JSON-escaped newline).
+const KEPT_ARGUMENTS_COMMANDS                      = new Set(DELTA_COMMANDS);
+// Those whose description is just the command (no child plan printed under it), so it is small
+// enough to decode whole when it spans decoder chunks. SaveIntoDataSourceCommand prints the whole
+// query under it, which can be as large as any other plan description.
+const SMALL_DESCRIPTION_COMMANDS                      = new Set(
+  [...KEPT_ARGUMENTS_COMMANDS].filter((command) => command !== 'SaveIntoDataSourceCommand'),
+);
+const PLAN_HEADER = '== Physical Plan ==\\nExecute ';
+// The kept form once JSON-parsed: `<Command>\nArguments: <line>`.
+const KEPT_ARGUMENTS = /^[A-Za-z]+\nArguments: /;
 
 // Index of the closing quote of the JSON string whose content starts at `valueStart`: the first
 // quote preceded by an even number of backslashes. -1 when the string is unterminated.
@@ -1326,7 +1371,48 @@ function closingQuoteAt(buf            , limit        , carried        )
   return -1;
 }
 
-// Returns `line` with the physicalPlanDescription string value emptied, or `line` unchanged when
+// The command named by a description that starts at text[valueStart] with PLAN_HEADER, when text
+// holds the whole name; null otherwise.
+function describedCommand(text        , valueStart        )                {
+  if (!text.startsWith(PLAN_HEADER, valueStart)) return null;
+  const nameStart = valueStart + PLAN_HEADER.length;
+  let nameEnd = nameStart;
+  while (nameEnd < text.length && /[A-Za-z]/.test(text[nameEnd])) nameEnd++;
+  return nameEnd < text.length ? text.slice(nameStart, nameEnd) : null;
+}
+
+// What to do with an open description value whose first bytes are text[valueStart...]: 'keep' it
+// (a small Delta command description), 'skip' it, or 'wait' when too little has arrived to tell.
+function descriptionDisposition(text        , valueStart        )                           {
+  const seen = text.slice(valueStart, valueStart + PLAN_HEADER.length);
+  if (!PLAN_HEADER.startsWith(seen)) return 'skip';
+  if (seen.length < PLAN_HEADER.length) return 'wait';
+  const command = describedCommand(text, valueStart);
+  if (command === null) return 'wait'; // the command name is still arriving
+  return SMALL_DESCRIPTION_COMMANDS.has(command) ? 'keep' : 'skip';
+}
+
+// The `<Command>\nArguments: <line>` part (still JSON-escaped) of the root command of a plan
+// description whose value spans line[valueStart, quote), or null when it is not one of
+// KEPT_ARGUMENTS_COMMANDS. SaveIntoDataSourceCommand is kept only for the Delta source: its other
+// providers (JDBC) print option maps that this tool has no use for.
+function keptCommandArguments(line        , valueStart        , quote        )                {
+  const command = describedCommand(line, valueStart);
+  if (command === null || !KEPT_ARGUMENTS_COMMANDS.has(command)) return null;
+  const key = `${command}\\nArguments: `;
+  const at = line.indexOf(key, valueStart + PLAN_HEADER.length + command.length);
+  if (at === -1 || at >= quote) return null;
+  // The line ends at the first escaped newline: a backslash starts a two-char escape.
+  let end = at + key.length;
+  while (end < quote && !(line.charCodeAt(end) === BACKSLASH && line.charCodeAt(end + 1) === LOWER_N)) {
+    end += line.charCodeAt(end) === BACKSLASH ? 2 : 1;
+  }
+  const kept = line.slice(at, Math.min(end, quote));
+  return command === 'SaveIntoDataSourceCommand' && !kept.includes('DeltaDataSource') ? null : kept;
+}
+
+// Returns `line` with the physicalPlanDescription string value emptied (or cut to the kept
+// arguments line, see keptCommandArguments), or `line` unchanged when
 // the key isn't found in Spark's compact form. The key pattern can't match inside another JSON
 // string: there its quotes would be backslash-escaped. buildChunkDecoder already empties a value
 // that spans chunks, so here the value is empty or within one chunk.
@@ -1339,7 +1425,7 @@ export function stripPlanDescription(line        )         {
   const quote = closingQuoteIndex(line, valueStart);
   // Unterminated string (a truncated line): leave it for JSON.parse to reject.
   if (quote === -1) return line;
-  return line.slice(0, valueStart) + line.slice(quote);
+  return line.slice(0, valueStart) + (keptCommandArguments(line, valueStart, quote) ?? '') + line.slice(quote);
 }
 
 const TASK_END_PREFIX = '{"Event":"SparkListenerTaskEnd",';

package/vendor-core/event-schemas.js

@@ -321,13 +321,15 @@ export const TaskEndEventSchema = z.object({
 export const SqlExecutionStartEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart'),
   executionId: z.number(),
+  rootExecutionId: z.number().optional(),
   description: z.string().optional(),
+  physicalPlanDescription: z.string().optional(),
   time: z.number(),
   sparkPlanInfo: SparkPlanInfoFieldSchema,
 });
 
 // applyAdaptiveExecutionUpdate: AQE re-plans mid-query and re-emits sparkPlanInfo for the same
-// executionId; last write wins. Neither SQL schema declares physicalPlanDescription: nothing reads
+// executionId; last write wins. This schema does not declare physicalPlanDescription: nothing reads
 // it, so zod strips it (dispatchLine already empties it before JSON.parse, see stripPlanDescription).
 export const SqlAdaptiveExecutionUpdateEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate'),

package/vendor-core/evidence-report.js

@@ -180,29 +180,29 @@ export const EVIDENCE_SCHEMA_VERSION         = 5;
 // module (stageShape's totalCores, utilization's unrounded fraction) is left off both, so it never
 // reaches the report; adding, renaming or dropping a key here changes the report contract.
 const EVIDENCE_KEYS = {
-  skew: [],
+  skew: ['origin'],
   stageShape: ['rule'],
-  shuffle: [],
-  partitionSizing: ['rule'],
-  spill: ['spillMagnitude'],
+  shuffle: ['partitions'],
+  partitionSizing: ['rule', 'origin', 'partitions'],
+  spill: ['spillMagnitude', 'reads'],
   gc: ['direction'],
   slowHost: ['variant', 'host', 'hostTaskShare', 'hostMeanMs', 'dimension', 'executorId', 'execMaxValue'],
-  stageSlowness: [],
+  stageSlowness: ['reads'],
   stageFailed: ['variant', 'numTasks', 'memoryBytesSpilled', 'failedTaskDetails'],
   failures: ['failedTasks', 'dominantReason', 'dominantError', 'failureGroups', 'otherFailedTasks'],
-  straggler: ['unit', 'speculativeTasks', 'stragglerCount'],
+  straggler: ['unit', 'speculativeTasks', 'stragglerCount', 'origin'],
   speculationWaste: [],
   retryWaste: ['numTasks', 'memoryBytesSpilled', 'retriedTaskDetails'],
-  tinyTask: [],
+  tinyTask: ['reads'],
   incompleteRun: [],
-  coldStart: [],
+  coldStart: ['dynamicAllocation'],
   utilization: ['cpuUtilizationPct'],
   memoryUtilization: ['variant', 'rule', 'executorId', 'heap', 'dataUnavailable'],
   cacheUtilization: [
     'variant', 'rddId', 'rddName', 'memorySize', 'diskSize', 'numCachedPartitions', 'numPartitions', 'dataUnavailable',
   ],
   coreLocality: ['nonLocalTaskCount'],
-  autoscalingChurn: ['shortLivedExecutorCount'],
+  autoscalingChurn: ['shortLivedExecutorCount', 'dynamicAllocation'],
   cachingOpportunity: ['variant', 'relation', 'format', 'relations', 'operator', 'executionIds', 'totalReadBytes'],
   jobFailureRate: ['failedJobs', 'totalJobs', 'failedTasks', 'totalTasks', 'avgJobDurationMs', 'taskFailureRate'],
   configAudit: ['property'],
@@ -211,8 +211,8 @@ const EVIDENCE_KEYS = {
     'groupIndex',
   ],
   smallFiles: ['executionId', 'stageIds', 'fileCount', 'direction', 'nodeName'],
-  underBroadcast: ['executionId', 'stageIds', 'largerSideBytes'],
-  overBroadcast: ['executionId', 'stageIds'],
+  underBroadcast: ['executionId', 'stageIds', 'largerSideBytes', 'broadcastThreshold'],
+  overBroadcast: ['executionId', 'stageIds', 'broadcastThreshold'],
 }                                                                                    ;
 
 // The other direction: an evidence field EVIDENCE_KEYS doesn't list fails here.
@@ -405,13 +405,13 @@ function verdictJson(model                 )              {
 // A WeakMap needs no invalidation: once mcp-tools.ts evicts the appModel, this entry is collectible.
 // Each cache is split first by the overrides object the report ran under (one fixed, frozen object
 // per CLI invocation or MCP server process; DEFAULT_THRESHOLDS for the specification's).
-                                                                          
+                                                                                 
 const DEFAULT_THRESHOLDS = {};
 const jsonCache              = new WeakMap();
 // The redacted report, keyed by the unredacted appModel it was built from.
 const redactedJsonCache              = new WeakMap();
 
-function cacheFor(cache             , thresholds                                )                                        {
+function cacheFor   (cache                , thresholds                                )                       {
   const key = thresholds ?? DEFAULT_THRESHOLDS;
   let byModel = cache.get(key);
   if (!byModel) {
@@ -421,14 +421,23 @@ function cacheFor(cache             , thresholds
   return byModel;
 }
 
-function runFindings(appModel          , thresholds                                )                                            {
+const findingsCache                                                         = new WeakMap();
+
+/** The run's detector catalog and config audit, computed once per run and overrides so the report
+ * and the blocks built beside it (run-output.ts) share one analysis. */
+export function runFindings(appModel          , thresholds                                )                                            {
+  const cache = cacheFor(findingsCache, thresholds);
+  const cached = cache.get(appModel);
+  if (cached) return cached;
   const { app, stages, executors, sql, jobs, runAggregates } = appModel;
   const catalog = analyze(
     app, stages, executors?.added ?? [], executors?.removed ?? [],
     jobs ?? new Map(), sql ?? new Map(),
     runAggregates ?? null, { thresholds },
   );
-  return { catalog, config: auditConfig(app) };
+  const result = { catalog, config: auditConfig(app) };
+  cache.set(appModel, result);
+  return result;
 }
 
 // Redacts the model and findings before the report derives any text from them, the same order

package/vendor-core/finding-presentation.js

@@ -8,7 +8,7 @@
 // detectors.ts is type-only, so it is erased at build time. A detector's scope, order and emits
 // list stay on its DETECTORS entry; renderers get them through detectorInfoByType().
                                                    
-                                                      
+                                                                  
                                                          
 
                                                              
@@ -43,7 +43,9 @@ const SKEW_JOIN_ALREADY_ON = 'AQE skew-join handling is already on, so salt the
 const SKEW_JOIN_AQE_OFF = 'AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise salt the key or repartition on a better key.';
 
 // The skew-join generic line, worded per the row's remediation: AQE logged off, switch already on, or neither.
-function skewJoinGeneric(f                                 , unset        )         {
+function skewJoinGeneric(f                                                      , unset        )         {
+  if (f.origin === 'inputScan') return 'Uneven input files: compact small files or split large ones (lower spark.sql.files.maxPartitionBytes).';
+  if (f.origin === 'other') return 'Work is uneven across tasks: salt the key or repartition on a better key.';
   if (f.remediation?.some((r) => r.key === 'spark.sql.adaptive.enabled')) return SKEW_JOIN_AQE_OFF;
   return switchAlreadyOn(f, SKEW_JOIN_KEY) ? SKEW_JOIN_ALREADY_ON : unset;
 }
@@ -120,8 +122,9 @@ export const FINDING_PRESENTATION
     tag: 'TINY',
     thresholdSummary: (t) => `${t.minTasks}+ tasks with a median of ${t.maxP50}ms or less and a P95 of ${t.maxP95}ms or less`,
     actionLabel: () => 'Coalesce small tasks',
-    // The shuffle-vs-no-shuffle fix isn't a Finding field, so one sentence covers both.
-    genericRecommendation: () => 'Scheduler overhead may dominate: lower spark.sql.shuffle.partitions, or coalesce down to fewer, larger tasks.',
+    genericRecommendation: (f) => (f.reads != null && f.reads !== 'shuffle'
+      ? 'Scheduler overhead may dominate: coalesce down to fewer, larger tasks.'
+      : 'Scheduler overhead may dominate: lower spark.sql.shuffle.partitions, or coalesce down to fewer, larger tasks.'),
   },
 
   shuffle: {
@@ -129,7 +132,14 @@ export const FINDING_PRESENTATION
     tag: 'SHFL',
     thresholdSummary: (t) => `stage shuffle read above ${t.minBytes / 1048576} MiB`,
     actionLabel: () => 'Reduce shuffle size',
-    genericRecommendation: () => 'Raise spark.sql.shuffle.partitions, or use a broadcast join for the smaller side.',
+    genericRecommendation: (f) => {
+      switch (f.partitions) {
+        case 'sufficient': return 'The tasks are already a good size, so more partitions will not help: use a broadcast join for the smaller side.';
+        case 'aqeCoalesced': return 'AQE coalesced the shuffle: lower spark.sql.adaptive.advisoryPartitionSizeInBytes, or use a broadcast join for the smaller side.';
+        case 'ownPartitioning': return "spark.sql.shuffle.partitions is already high enough, so raise this stage's own partition count (its repartition(n) or RDD parallelism), or use a broadcast join for the smaller side.";
+        default: return 'Raise spark.sql.shuffle.partitions, or use a broadcast join for the smaller side.';
+      }
+    },
   },
   partitionSizing: {
     name: 'partition sizing',
@@ -146,7 +156,9 @@ export const FINDING_PRESENTATION
     genericRecommendation(f) {
       switch (f.rule) {
         case 'shufflePartitionSkew': return skewJoinGeneric(f, 'For join skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.');
-        case 'lowShuffleParallelism': return switchAlreadyOn(f, 'spark.sql.shuffle.partitions')
+        case 'lowShuffleParallelism':
+          if (f.partitions === 'aqeCoalesced') return 'AQE coalesced the shuffle into few tasks: lower spark.sql.adaptive.advisoryPartitionSizeInBytes so each partition is smaller.';
+          return switchAlreadyOn(f, 'spark.sql.shuffle.partitions')
           ? "spark.sql.shuffle.partitions is already high enough, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller."
           : 'Raise spark.sql.shuffle.partitions so each partition is smaller.';
         case 'maxPartitionTooBig': return 'Repartition to break up the oversized partition before this stage.';
@@ -161,7 +173,9 @@ export const FINDING_PRESENTATION
     thresholdSummary: (t) => `single-task disk spill above ${t.singleTaskDiskGiB} GiB`,
     actionLabel: () => 'Reduce spill',
     // The skew/volume classification isn't a Finding field, so one sentence covers both.
-    genericRecommendation: () => 'If the spill is skew-driven, fix task skew first: adding memory will not help. Otherwise raise spark.sql.shuffle.partitions or increase executor memory.',
+    genericRecommendation: (f) => (f.reads != null && f.reads !== 'shuffle'
+      ? 'If the spill is skew-driven, fix task skew first: adding memory will not help. Otherwise increase executor memory or process less data per task.'
+      : 'If the spill is skew-driven, fix task skew first: adding memory will not help. Otherwise raise spark.sql.shuffle.partitions or increase executor memory.'),
   },
 
   gc: {
@@ -219,14 +233,18 @@ export const FINDING_PRESENTATION
     tag: 'SLOW',
     thresholdSummary: () => 'a stage running far longer than its peers, not attributable to a single slow host',
     actionLabel: () => 'Profile slow stage',
-    genericRecommendation: () => 'Often a partition-count problem: raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.',
+    genericRecommendation: (f) => f.reads === 'input'
+      ? 'Often too few or too uneven input partitions: check input file sizes and lower spark.sql.files.maxPartitionBytes, or look for a large per-task data volume driving heavy spill.'
+      : f.reads === 'other'
+        ? 'Check what the stage computes and for a large per-task data volume driving heavy spill.'
+        : 'Often a partition-count problem: raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.',
   },
   straggler: {
     name: 'straggling task',
     tag: 'STRAG',
     thresholdSummary: () => 'one or more tasks finishing far after the rest of their stage',
     actionLabel: () => 'Fix stragglers',
-    genericRecommendation: () => 'Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue. If a skewed key is the real cause, that is a candidate for AQE\'s skew-join handling.',
+    genericRecommendation: (f) => `Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue. If uneven data is the cause: ${skewJoinGeneric(f, 'for join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.')}`,
   },
   speculationWaste: {
     name: 'speculation waste',
@@ -240,7 +258,9 @@ export const FINDING_PRESENTATION
     tag: 'COLD',
     thresholdSummary: (t) => `the first stage waiting over ${t.gapSeconds}s for an executor`,
     actionLabel: () => 'Pre-warm cluster',
-    genericRecommendation: () => 'Keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it does not scale up from zero.',
+    genericRecommendation: (f) => (f.dynamicAllocation === 'off'
+      ? 'Keep a warm pool of idle executors: dynamic allocation is off, so no executor-count property applies.'
+      : 'Keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it does not scale up from zero.'),
   },
 
   memoryUtilization: {
@@ -322,7 +342,9 @@ export const FINDING_PRESENTATION
     tag: 'CHRN',
     thresholdSummary: (t) => `over ${shareLabel(t.warningPct)} of executors living under ${t.shortLivedMs / 60000} minutes`,
     actionLabel: () => 'Reduce autoscaling churn',
-    genericRecommendation: () => 'This looks like wasteful re-provisioning rather than normal scale-down: consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.',
+    genericRecommendation: (f) => (f.dynamicAllocation === 'off'
+      ? 'Dynamic allocation is off, so the churn comes from executors lost or preempted: check the cluster manager\'s preemption and executor-loss logs.'
+      : 'This looks like wasteful re-provisioning rather than normal scale-down: consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.'),
   },
 
   configAudit: CONFIG_AUDIT_PRESENTATION,
@@ -349,14 +371,18 @@ export const FINDING_PRESENTATION
     tag: 'PLAN',
     thresholdSummary: () => 'a join below the configured size floor that skipped broadcast',
     actionLabel: () => 'Use broadcast join',
-    genericRecommendation: () => 'This could have been a broadcast join: consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.',
+    genericRecommendation: (f) => (f.broadcastThreshold === 'notLimiting'
+      ? 'The threshold already admits the smaller side, so it is not what stopped the broadcast: consider a broadcast() hint or collecting table statistics.'
+      : 'This could have been a broadcast join: consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.'),
   },
   overBroadcast: {
     name: 'oversized broadcast join',
     tag: 'PLAN',
     thresholdSummary: (t) => `a broadcast over ${t.overBroadcastBytes / 1073741824} GiB`,
     actionLabel: () => 'Fix oversized broadcast',
-    genericRecommendation: (f) => (switchAlreadyOn(f, 'spark.sql.autoBroadcastJoinThreshold')
+    genericRecommendation: (f) => (f.broadcastThreshold === 'notLimiting'
+      ? 'The configured threshold is below this broadcast, so remove the broadcast() hint that forced it.'
+      : switchAlreadyOn(f, 'spark.sql.autoBroadcastJoinThreshold')
       ? 'Automatic broadcast is already disabled, so remove the broadcast() hint that forced it.'
       : 'Check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.'),
   },

package/vendor-core/finding-types.js

@@ -79,7 +79,22 @@
 
 // ── Per-stage detectors ─────────────────────────────────────────────────────
 
-                                                
+/** What a skewed stage reads, which decides the fix: 'shuffleJoin' (a shuffle feeding a join, where
+ * AQE skew-join handling applies), 'inputScan' (uneven input files) or 'other'. */
+                                                               
+
+/** What a stage reads, by the dominant side: a shuffle, input files, or neither. */
+                                                       
+
+/** Why a shuffle finding's partition-count advice is or is not given: 'raise' (the property limits
+ * the stage), 'sufficient' (tasks are already a good size), 'aqeCoalesced' (AQE merged the
+ * partitions, so the advisory size is the lever) or 'ownPartitioning' (the property is already high
+ * enough, so the stage's own repartition(n) or RDD parallelism is). */
+                                                                                            
+
+                               
+                      
+ 
                                                                             
 
                                      
@@ -90,16 +105,24 @@
                       
  
 
-                                                   
+                                  
+                                 
+ 
                                                                                      
 
                                           
                                                                                 
+                                                    
+                      
+                                                                                                           
+                                 
  
                                                                                                              
 
                                 
                                                 
+                                                                              
+                     
  
                                                                                
 
@@ -124,7 +147,10 @@
  
                                                                                         
 
-                                                         
+                                        
+                                                                                     
+                     
+ 
                                                                                                        
 
                                       
@@ -151,6 +177,8 @@
                         
                            
                          
+                                                                                     
+                      
  
                                                                                            
 
@@ -167,7 +195,10 @@
                     
  
 
-                                                    
+                                   
+                                                                                  
+                     
+ 
                                                                                         
 
 // ── App-level detectors ─────────────────────────────────────────────────────
@@ -176,7 +207,10 @@
 // valueText is always 'missing': the ApplicationEnd event the log lacks.
                                                                                                     
 
-                                                     
+                                    
+                                                                                                
+                                   
+ 
                                                                                            
 
                                       
@@ -231,6 +265,8 @@
 
                                            
                                   
+                                                                                           
+                                   
  
                                                                                                                 
 
@@ -296,13 +332,21 @@
                    
  
                                                                                                               
+
+/** How the effective spark.sql.autoBroadcastJoinThreshold relates to the finding: 'limits' (the
+ * property decided it, or is unknown), 'notLimiting' (already admits, or already below, the
+ * broadcast, so something else decided it) or 'disabled' (-1). */
+                                                                       
 
                                                                      
                           
+                                          
  
                                                                                                                           
 
-                                                        
+                                                                    
+                                          
+ 
                                                                                                                        
 
 // ── Unions ──────────────────────────────────────────────────────────────────

package/vendor-core/mcp-server-factory.js

@@ -6,6 +6,7 @@ import {
 } from './mcp-tools.js';
 import { listRuns } from './list-runs.js';
                                                          
+import { MAX_NORMALIZE_PATTERN_LENGTH, MAX_NORMALIZE_PATTERNS } from './stage-alignment.js';
 
 const sourceSchema = z.union([
   z.object({ path: z.string() }),
@@ -71,7 +72,7 @@ export function createMcpServer({ thresholds }
   }, (params) => toolResult(listRuns(params)));
 
   server.registerTool('diagnose_run', {
-    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, and the checks the log lacked the data to run. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
+    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, the checks the log lacked the data to run, and the writeTargets, metrics and effectiveConf blocks of the CLI report. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
     inputSchema: {
       ...runRefSchema, redact: z.boolean().optional(),
       include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
@@ -91,15 +92,19 @@ export function createMcpServer({ thresholds }
   ));
 
   server.registerTool('compare_runs', {
-    description: 'Compare two runs: categorized findings delta and metric deltas.',
+    description: 'Compare two runs: the verdict, metric deltas (metrics), categorized findings delta (findings) and the stage pairs (stagePairs) with the share of executor run time they cover (runtimeCoverage). confidence is ok, low or insufficient. metricDeltas and findingsDelta repeat metrics and findings under their earlier names and will be removed.',
     inputSchema: {
       runIdA: z.string().optional(), sourceA: sourceSchema.optional(),
       ...secondRunRefSchema,
       redact: z.boolean().optional(),
+      include: z.array(z.enum(['stagePairs'])).optional()
+        .describe('Extra blocks to return. stagePairs: one row per paired stage with its deltas, which can be large.'),
+      normalizePath: z.array(z.string().max(MAX_NORMALIZE_PATTERN_LENGTH)).max(MAX_NORMALIZE_PATTERNS).optional()
+        .describe('Regular expressions whose every match in a plan node\'s text is replaced with a fixed token before stages are paired, so run-specific text (an output directory that differs per run) does not keep the same stage from pairing. A generic pattern, not only for paths. Affects stage pairing, runtimeCoverage and confidence, never findings. A pattern that backtracks catastrophically can stall the server: keep them simple and anchored.'),
       ...formatSchema,
     },
-  }, ({ runIdA, sourceA, runIdB, sourceB, redact, format }) => toolResultWithMarkdown(
-    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds }),
+  }, ({ runIdA, sourceA, runIdB, sourceB, redact, include, normalizePath, format }) => toolResultWithMarkdown(
+    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds, normalizePath, include }),
   ));
 
   server.registerTool('evaluate_budgets', {
@@ -112,14 +117,20 @@ export function createMcpServer({ thresholds }
       sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced). When given, the absolute budgets (maxRuntimeMs etc.) are evaluated on this run.'),
       runIdB: secondRunRefSchema.runIdB.describe('Same as `sourceB`, referencing an already-resolved run by id.'),
       maxRegressionPct: z.number().optional(), regressionMetric: z.string().optional(), failOnIntroduced: z.string().optional(),
+      regressionBudgets: z.array(z.object({ metric: z.string(), maxPct: z.number().nonnegative() })).optional()
+        .describe('Further regression budgets, one per metric (the CLI\'s repeated --regression-budget). A metric can be budgeted once across this list and maxRegressionPct/regressionMetric.'),
+      stageRegressionBudgets: z.array(z.object({ metric: z.string(), maxPct: z.number().nonnegative() })).optional()
+        .describe('Per paired stage (needs sourceB/runIdB): fail when any eligible pair\'s metric (executorRunTime, executorCpuTime, memoryBytesSpilled, diskBytesSpilled, shuffleReadBytes, shuffleWriteBytes) regressed by more than maxPct. One entry per metric. Inconclusive when no stage pairs.'),
+      stageQualities: z.array(z.enum(['exact', 'structural', 'aligned'])).optional()
+        .describe('Pair qualities the stage budgets read. Default exact and structural: an aligned pair may compare different work.'),
     },
   }, ({
     source, runId, runIdB, sourceB,
     maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct,
-    maxRegressionPct, regressionMetric, failOnIntroduced,
+    maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets, stageRegressionBudgets, stageQualities,
   }) => toolResult(evaluateBudgetsForRun(
     { source, runId },
-    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced },
+    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets, stageRegressionBudgets, stageQualities },
     (runIdB || sourceB) ? { runId: runIdB, source: sourceB } : undefined,
     { thresholds },
   )));