sparkforensics-server 0.5.0 → 0.5.1

package/public/index.html

@@ -23,12 +23,12 @@
           document.documentElement.setAttribute('data-theme', 'light');
       } catch (_) {}
     </script>
-    <script type="module" crossorigin src="./assets/index-DaC8A2FS.js"></script>
+    <script type="module" crossorigin src="./assets/index-DCeWge12.js"></script>
     <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
     <link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
-    <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-B0sEms9Z.js">
-    <link rel="modulepreload" crossorigin href="./assets/react-vendor-BAKt3vny.js">
-    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-Ds0hcZFn.js">
+    <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-BYl-AzN5.js">
+    <link rel="modulepreload" crossorigin href="./assets/react-vendor-DEdfrRTJ.js">
+    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-B0siMVdg.js">
     <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-DLioOiRN.css">
     <link rel="stylesheet" crossorigin href="./assets/index-BcqTLKFp.css">
   </head>

package/vendor-core/analyzer.js

@@ -90,7 +90,7 @@ export function findingId(f         )         {
 
 // skew (either branch) and straggler both claim the stage's replayed tail recovery
 // (tailReplayRecoveryMs via tailRecoveryMs): the same slow-task tail reported by two detectors
-// (see "Overlap caveat: skew / straggler" in impact-estimation.md). skew's branch only changes
+// (see "Overlap caveat: skew / straggler" in impact-estimation/caveats-tuning-and-coverage.md). skew's branch only changes
 // the fallback single-task delta on a stage without the replay, so every skew + straggler pair
 // on a stage is flagged. Flags both sides via validationRequired (rather than suppressing
 // either) so neither finding's own diagnostic value is lost; the flag rides the same

package/vendor-core/api-merge-writes.js

@@ -0,0 +1,80 @@
+// DeltaTable API merges run no command node: each internal query is its own root execution, tied
+// together only by a "MERGE operation" description and adjacent ids. One DeltaMerge write is
+// reported per run of such executions, with a path only when no other merge can be mixed into it.
+import { soleLogPath } from './delta-log-paths.js';
+                                               
+                                                      
+
+                                                                                                              
+
+const API_MERGE_COMMAND = 'DeltaMerge';
+
+// Delta labels its MERGE sub-queries "MERGE operation - <phase>", prefixed "Delta: " in some.
+const MERGE_DESCRIPTION = /^(?:Delta: )?MERGE operation\b/;
+
+// The sub-queries that write: "writing new files ...", "Rewriting N files". A group without one
+// only scanned, and a scan may read the merge's source rather than its target.
+const MERGE_WRITE_PHASE = /\b(?:re)?writing\b/i;
+
+                                                                          
+
+// Selected executions have a MERGE description. A SQL MERGE's own sub-queries carry the same
+// description but belong to the MergeIntoCommand execution they share a root with, which is
+// attributed through its own node (resolveDeltaCommandTarget), so they are left out. A sub-query
+// of an API merge may have another of its executions as its root, so a root alone does not exclude.
+function selectMergeExecutions(sql                             )                   {
+  const isSqlMergeRoot = (id                    )          =>
+    id !== undefined && /^Execute\s+MergeIntoCommand\b/.test(sql.get(id)?.planTree?.name ?? '');
+  return [...sql.values()]
+    .filter((exec) => MERGE_DESCRIPTION.test(exec.description ?? '') && !isSqlMergeRoot(exec.rootExecutionId))
+    .sort((a, b) => a.id - b.id);
+}
+
+// A run of consecutive execution ids. Nothing in the log ties the members of one merge together,
+// so adjacency is a guess, and a wrong one fuses or splits merges. Each way it can go wrong is
+// marked `unclean` instead of attributed:
+//   - the ids of two merges that ran concurrently interleave, so their start-time spans overlap;
+//   - a start time is missing, so the span is unknown.
+//   - no member writes, so the one path it names may be a source's.
+// Merges that ran back to back with no other execution between them still fuse into one run; the
+// run then names two paths and gets no target.
+function groupMerges(selected                  )               {
+  const groups               = [];
+  for (const exec of selected) {
+    const last = groups[groups.length - 1];
+    if (last && last.members[last.members.length - 1].id + 1 === exec.id) last.members.push(exec);
+    else groups.push({ members: [exec], unclean: null });
+  }
+  const spans = groups.map(({ members }) => {
+    const starts = members.map((m) => m.startTime);
+    return starts.every((t)              => typeof t === 'number') ? [Math.min(...starts), Math.max(...starts)] : null;
+  });
+  groups.forEach((group, i) => {
+    const mine = spans[i];
+    if (mine === null) { group.unclean = 'start time missing'; return; }
+    const overlaps = spans.some((other, j) => j !== i && other !== null && other[0] <= mine[1] && mine[0] <= other[1]);
+    if (overlaps) group.unclean = 'overlaps another merge in time';
+    else if (!group.members.some((m) => MERGE_WRITE_PHASE.test(m.description ?? ''))) group.unclean = 'no write phase';
+  });
+  return groups;
+}
+
+/** One DeltaMerge write per run of API merge executions; null target unless one path is certain. */
+export function apiMergeWrites(sql                             )                {
+  return groupMerges(selectMergeExecutions(sql)).map((group) => {
+    const first = group.members[0].id;
+    const last = group.members[group.members.length - 1].id;
+    const resolved = group.unclean === null ? soleLogPath(group.members) : null;
+    const span = first === last ? `${first}` : `${first}-${last}`;
+    return {
+      sqlExecutionId: first,
+      nodeId: null,
+      command: API_MERGE_COMMAND,
+      recognized: true,
+      kind: resolved?.kind ?? null,
+      target: resolved?.target ?? null,
+      outputRows: null,
+      raw: `Delta MERGE operation, executions ${span}${group.unclean ? ` (${group.unclean})` : ''}`,
+    };
+  });
+}

package/vendor-core/comparison-output.js

@@ -0,0 +1,49 @@
+import { buildComparison,                                             } from './run-comparison.js';
+import { comparisonVerdict,                            } from './comparison-verdict.js';
+import { redactComparison } from './redact.js';
+                                                    
+
+                                                                                             
+
+// The comparison a surface reports: the CLI's `comparison` object and the shared part of MCP
+// compare_runs. A field added here reaches every surface.
+// `stagePairs` is the one large field (a row per paired stage), so the `summary` view leaves it out.
+                                                                 
+                           
+                                                                                            
+                                                                                    
+                                                   
+
+/** `full` carries every field (the CLI); `summary` leaves out `stagePairs` (MCP, which returns them
+ * only when asked). */
+                                                
+
+/** The reported projection of a comparison. Key order is the CLI's JSON order. */
+export function comparisonOutput(comparison                   , view                 = 'full')                   {
+  return {
+    verdict: comparisonVerdict(comparison),
+    confidence: comparison.confidence,
+    reason: comparison.reason,
+    matchedCoverage: comparison.matchedCoverage,
+    runtimeCoverage: comparison.runtimeCoverage,
+    metrics: comparison.metrics,
+    findings: comparison.findings,
+    comparisonSchemaVersion: comparison.comparisonSchemaVersion,
+    ...(view === 'full' ? { stagePairs: comparison.stagePairs } : {}),
+    unmatched: comparison.unmatched,
+    replanned: comparison.replanned,
+    bookkeepingStageIds: comparison.bookkeepingStageIds,
+  };
+}
+
+/** Diffs the two runs and applies `redact` here, so no surface can skip it (stage names carry raw
+ * Spark text that can embed a host or IP). `comparison` is the (redacted) raw result for budgets
+ * and Markdown; `output` is the projection to serialize. */
+export function buildComparisonOutput(
+  baseline                    , candidate                    ,
+  { redact, normalizePath, view }                                                               = {},
+)                                                              {
+  const built = buildComparison(baseline, candidate, { normalizePath });
+  const comparison = redact ? redactComparison(built) : built;
+  return { comparison, output: comparisonOutput(comparison, view) };
+}

package/vendor-core/core-source-hash.txt

@@ -1 +1 @@
-ab36c5fdcdc1a32a4e6206459e843e8d2267a90e6f1de29671827315d81bc374
+6e0ac29f74ae092b14c5700dfa86d3273fce7d1af59005c6e2e450a91da92d5e

package/vendor-core/delta-commands.js

@@ -0,0 +1,13 @@
+// Delta write commands whose root node is a bare name in sparkPlanInfo on real logs. The parser keeps
+// their `Arguments:` line (event-handlers.ts) and delta-targets.ts reads their target from it; the
+// handler table there is typed by this list, so a command added here must get a handler.
+export const DELTA_COMMANDS = [
+  'MergeIntoCommand', 'UpdateCommand', 'DeleteCommand', 'WriteIntoDelta', 'WriteIntoDeltaCommand',
+  'SaveIntoDataSourceCommand',
+]         ;
+
+                                                           
+
+export function isDeltaCommand(name        )                       {
+  return (DELTA_COMMANDS                     ).includes(name);
+}

package/vendor-core/delta-log-paths.js

@@ -0,0 +1,42 @@
+// The `<table path>/_delta_log` paths that plans name: the only trace of which Delta table a command
+// or API merge touched when its plan carries no target. A scan may read a source too, so callers
+// take a path only when exactly one is named.
+import { walkPlanTree } from './plan-tree-walk.js';
+import { isCut,                  } from './write-target-args.js';
+                                               
+
+// A URI or absolute path before `/_delta_log`, as printed in `... State #0 - <path>/_delta_log[cols]`
+// scans and `DeltaLogFileIndex [<path>/_delta_log/<file>, ...]`.
+const DELTA_LOG_PATH = /((?:[A-Za-z][A-Za-z0-9+.-]*:\/{1,3}|\/)[^\s,[\]()#]+?)\/_delta_log(?=[/[\],\s)]|$)/g;
+
+                                                              
+
+// Every distinct table path named in the plans; `unreliable` when a plan is missing or a path cut.
+function deltaLogPaths(executions                                  )           {
+  const paths = new Set        ();
+  let unreliable = false;
+  for (const exec of executions) {
+    if (!exec.planTree) { unreliable = true; continue; }
+    walkPlanTree(exec.planTree, (node) => {
+      for (const text of [node.name, node.detail ?? '']) {
+        const mentions = text.split('/_delta_log').length - 1;
+        if (mentions === 0) continue;
+        const matches = [...text.matchAll(DELTA_LOG_PATH)];
+        // A `/_delta_log` that is not read as a path is a table this set cannot name.
+        if (matches.length < mentions) unreliable = true;
+        for (const m of matches) {
+          if (isCut(m[1])) unreliable = true;
+          else paths.add(m[1].replace(/\/+$/, ''));
+        }
+      }
+    }, { dedupe: true });
+  }
+  return { paths, unreliable };
+}
+
+/** The one table path the plans of `executions` name, or null for none, several or an unreadable plan. */
+export function soleLogPath(executions                                  )                     {
+  const { paths, unreliable } = deltaLogPaths(executions);
+  const [path] = paths;
+  return !unreliable && paths.size === 1 ? { kind: 'path', target: path } : null;
+}

package/vendor-core/delta-targets.js

@@ -0,0 +1,127 @@
+// Delta write targets that the plan node's simpleString does not carry. A Delta command node is a
+// bare name in sparkPlanInfo on real logs; its target is read from the `Arguments:` line the parser
+// keeps (SqlExecution.commandArguments, one reader per command in COMMANDS), and for a command that
+// always scans its target (MERGE, UPDATE, DELETE) from the single `_delta_log` path of the executions
+// sharing its root. A write command may read another Delta table as its source, so it never takes
+// that step. Otherwise the target is null: a rule that cannot name exactly one target names none.
+import { isDeltaCommand,                   } from './delta-commands.js';
+import { soleLogPath } from './delta-log-paths.js';
+import { isCut, splitArgs, tableName,                  } from './write-target-args.js';
+                                                         
+                                                      
+
+                                                                                                       
+
+                                                                       
+
+/** Executions grouped by rootExecutionId, each group excluding the root itself. */
+export function indexByRoot(sql                               )                   {
+  const byRoot = new Map                            ();
+  for (const exec of sql.values()) {
+    if (exec.rootExecutionId === undefined || exec.rootExecutionId === exec.id) continue;
+    const list = byRoot.get(exec.rootExecutionId);
+    if (list) list.push(exec);
+    else byRoot.set(exec.rootExecutionId, [exec]);
+  }
+  return byRoot;
+}
+
+// The text after `<Command>\nArguments: `, or null when the execution kept none for this command.
+function argumentsOf(exec                  , command        )                {
+  const kept = exec.commandArguments;
+  const head = `${command}\nArguments: `;
+  return kept !== undefined && kept.startsWith(head) ? kept.slice(head.length) : null;
+}
+
+// A catalog table is printed as its quoted identifier, `catalog`.`db`.`t`, as one top-level
+// argument. Two or more such arguments (or none) leave the table unnamed. The session catalog is
+// implied by a Hive or V1 table name, so it is dropped: `spark_catalog`.`db`.`t` is db.t.
+const QUOTED_TABLE = /^`[^`]+`(?:\.`[^`]+`){1,2}$/;
+
+function tableFromArguments(args        )                     {
+  const candidates = splitArgs(args).filter((arg) => QUOTED_TABLE.test(arg.text));
+  if (candidates.length !== 1 || !candidates[0].terminated) return null;
+  const name = tableName(candidates[0].text);
+  if (!name || isCut(name)) return null;
+  const parts = name.split('.');
+  return { kind: 'table', target: parts.length === 3 && parts[0] === 'spark_catalog' ? parts.slice(1).join('.') : name };
+}
+
+// SaveIntoDataSourceCommand prints `<provider>@<hash>, [k=v, k2=v2], <mode>`. The target is the
+// one `path` option of a Delta provider.
+function pathFromSaveOptions(args        )                     {
+  const options = splitArgs(args)[1];
+  if (!options?.terminated || !/^\[[\s\S]*\]$/.test(options.text)) return null;
+  const body = options.text.slice(1, -1);
+  const keys = [...body.matchAll(/(?:^|, )([A-Za-z0-9_.-]+)=/g)];
+  const paths = keys.flatMap((key, i) => {
+    if (key[1].toLowerCase() !== 'path') return [];
+    return [body.slice(key.index  + key[0].length, i + 1 < keys.length ? keys[i + 1].index  : body.length)];
+  });
+  if (paths.length !== 1 || isCut(paths[0]) || /\*{5,}\(redacted\)/.test(paths[0])) return null;
+  return { kind: 'path', target: paths[0] };
+}
+
+                          
+                                                                       
+                                                  
+                                                                                                
+                       
+ 
+
+const tableCommand                 = { fromArguments: tableFromArguments, scansTarget: false };
+const scanningTableCommand                 = { ...tableCommand, scansTarget: true };
+
+// One entry per Delta command: a new command is added here (and to DELTA_COMMANDS), not in the resolver.
+const COMMANDS                                       = {
+  MergeIntoCommand: scanningTableCommand,
+  UpdateCommand: scanningTableCommand,
+  DeleteCommand: scanningTableCommand,
+  WriteIntoDelta: tableCommand,
+  WriteIntoDeltaCommand: tableCommand,
+  // Only the arguments say a save is a Delta one; a JDBC save with a Delta table in its source
+  // would otherwise take that table as its target.
+  SaveIntoDataSourceCommand: {
+    fromArguments: (args) => (args.includes('DeltaDataSource') ? pathFromSaveOptions(args) : null),
+    scansTarget: false,
+  },
+};
+
+/** Target of a Delta command whose simpleString names none; `node` must be the root of `exec`'s plan. */
+export function resolveDeltaCommandTarget(
+  command        , exec                  , node          , byRoot                  ,
+)                     {
+  if (!isDeltaCommand(command) || node !== exec.planTree) return null;
+  const handler = COMMANDS[command];
+  const args = argumentsOf(exec, command);
+  const found = args === null ? null : handler.fromArguments(args);
+  if (found || !handler.scansTarget) return found;
+  // Child executions are the root's own only when it is the root: a command run under another
+  // execution shares that execution's children, which may belong to unrelated work.
+  return exec.rootExecutionId === exec.id ? soleLogPath(byRoot.get(exec.id) ?? []) : null;
+}
+
+// An append or overwrite of an existing Delta table (saveAsTable, INSERT INTO or OVERWRITE) is an
+// `AppendDataExecV1` or `OverwriteByExpressionExecV1` whose first argument is the table object:
+// `DeltaTableV2(<session>,<path>,Some(CatalogTable(\nCatalog: c\nDatabase: d\nTable: t\n...)),...)`.
+// Its CatalogTable block names the table; a path-based table has `None` there and only the path.
+const DELTA_TABLE_V2 = /^\w+ DeltaTableV2\([^,\s]*,([^,\n]*),(?:Some\(CatalogTable\(\n(?:Catalog: (\S+)\n)?Database: (\S+)\nTable: (\S+)\n|None,)/;
+const SIMPLE_NAME = /^[A-Za-z0-9_$]+$/;
+
+/** Target of a V1-fallback write on a DeltaTableV2 table, or null when the detail has another shape. */
+export function parseDeltaTableV2(detail        )                     {
+  const m = DELTA_TABLE_V2.exec(detail);
+  if (!m) return null;
+  const [, path, catalog, database, table] = m;
+  if (table === undefined) return path && !isCut(path) ? { kind: 'path', target: path } : null;
+  if (![database, table, catalog ?? 'spark_catalog'].every((part) => SIMPLE_NAME.test(part))) return null;
+  // The session catalog is implied by a V1 name, so it is dropped (as for a Delta command).
+  const name = `${database}.${table}`;
+  return { kind: 'table', target: catalog === undefined || catalog === 'spark_catalog' ? name : `${catalog}.${name}` };
+}
+
+/** The append a Delta CTAS or RTAS runs under its own root: its staged table has no name. */
+export function isStagedDeltaWrite(write                                      )          {
+  return (write.command === 'AppendDataExecV1' || write.command === 'OverwriteByExpressionExecV1')
+    && /^\w+ \S*DeltaCatalog\$StagedDeltaTableV2@/.test(write.raw);
+}

package/vendor-core/event-handlers.js

@@ -18,6 +18,7 @@ import {
                   
                      
 } from './event-schemas.js';
+import { DELTA_COMMANDS } from './delta-commands.js';
 import { assertNever } from './assert-never.js';
 import { finalizeStage } from './stage-quantiles.js';
 import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
@@ -198,6 +199,9 @@ const MAX_TASK_SAMPLES = 20;
                     
                          
                      
+                           
+                                                                                  
+                            
                                                                                          
                                       
                                                                                                   
@@ -279,6 +283,10 @@ export function buildChunkDecoder() {
   // line again after every slice scanned about 10 GB on a 100 MB line.
   let pendingIsSqlEvent = false;
   let keyTail = '';
+  // Where the pending line's plan-description value starts, once its key is found but too little
+  // of the value has arrived to tell whether it is a small Delta command description (see
+  // descriptionDisposition); -1 otherwise.
+  let undecidedValueStart = -1;
   let skippingPlanDescription = false;
   // Length of the backslash run the skipped bytes ended with, which escapes a quote at the start
   // of the next chunk when odd.
@@ -312,11 +320,24 @@ export function buildChunkDecoder() {
       keyTail = '';
       appended = pending; // nothing of it was searched while it was shorter than the prefix
     }
-    const keyAt = findPlanKey(appended);
-    if (keyAt === -1) return; // the key may still arrive in a later chunk
+    let valueStart = undecidedValueStart;
+    if (valueStart === -1) {
+      const keyAt = findPlanKey(appended);
+      if (keyAt === -1) return; // the key may still arrive in a later chunk
+      valueStart = keyAt + PLAN_DESCRIPTION_KEY.length;
+    }
+    if (closingQuoteIndex(pending, valueStart) !== -1) { // complete: stripPlanDescription handles it
+      undecidedValueStart = -1;
+      pendingSettled = true;
+      return;
+    }
+    // A small kept description is not skipped: the line completes in a later chunk and
+    // stripPlanDescription keeps its arguments line.
+    const disposition = descriptionDisposition(pending, valueStart);
+    if (disposition === 'wait') { undecidedValueStart = valueStart; return; }
+    undecidedValueStart = -1;
     pendingSettled = true;
-    const valueStart = keyAt + PLAN_DESCRIPTION_KEY.length;
-    if (closingQuoteIndex(pending, valueStart) !== -1) return; // complete: stripPlanDescription empties it
+    if (disposition === 'keep') return;
     // Only a chunk whose last byte is a backslash carries a run over; any other last byte (such
     // as part of a split multibyte char) ends it.
     let run = 0;
@@ -378,6 +399,7 @@ export function buildChunkDecoder() {
       pending = pendingHead = appended = start < text.length ? text.substring(start) : '';
       pendingSettled = false;
       pendingIsSqlEvent = false;
+      undecidedValueStart = -1;
     }
     if (!pendingSettled && pending !== '') settlePending(buffer[buffer.length - 1], appended);
   }
@@ -1109,6 +1131,10 @@ export function startSqlExecution(event
     sparkPlanInfo,
     hadAdaptiveUpdate: false,
   };
+  if (event.rootExecutionId !== undefined) exec.rootExecutionId = event.rootExecutionId;
+  if (event.physicalPlanDescription && KEPT_ARGUMENTS.test(event.physicalPlanDescription)) {
+    exec.commandArguments = event.physicalPlanDescription;
+  }
   state.sqlExecutions.set(exec.id, exec);
   // A restarted execution carries a new plan: its next end must resolve it again.
   state.resolvedPlanExecutions.delete(exec.id);
@@ -1295,13 +1321,32 @@ const KNOWN_EVENT_TYPES                      = new Set(
 );
 
 // SQLExecutionStart and SQLAdaptiveExecutionUpdate carry `physicalPlanDescription`, Spark's text
-// rendering of the plan. Nothing reads it (the plan tree comes from sparkPlanInfo), yet on a real
-// 3.5 GB log it was 72% of the AQE-update bytes, which were themselves 73% of the log. Cutting its
+// rendering of the plan. The plan tree comes from sparkPlanInfo, yet on a real 3.5 GB log the
+// description was 72% of the AQE-update bytes, which were themselves 73% of the log. Cutting its
 // string value out before JSON.parse halves the parse cost of those lines.
+//
+// One exception: a Delta write command's node prints only its bare name in sparkPlanInfo, and its
+// target table appears only on its `Arguments:` line of the description. When the description is
+// that of one of KEPT_ARGUMENTS_COMMANDS (checked on its first bytes, so no other line pays for it),
+// the value is cut down to that one line, never kept whole. A value that spans decoder chunks is
+// still dropped unread (buildChunkDecoder), so the write target then falls back to its child paths.
 const SQL_UI_EVENT_PREFIX = '{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQL';
 const PLAN_DESCRIPTION_KEY = '"physicalPlanDescription":"';
 
-const QUOTE = 0x22, BACKSLASH = 0x5c, NEWLINE = 0x0a;
+const QUOTE = 0x22, BACKSLASH = 0x5c, NEWLINE = 0x0a, LOWER_N = 0x6e;
+
+// Delta commands whose root node carries no target in sparkPlanInfo. The description of a
+// command execution starts `== Physical Plan ==\nExecute <Command> (1)` (JSON-escaped newline).
+const KEPT_ARGUMENTS_COMMANDS                      = new Set(DELTA_COMMANDS);
+// Those whose description is just the command (no child plan printed under it), so it is small
+// enough to decode whole when it spans decoder chunks. SaveIntoDataSourceCommand prints the whole
+// query under it, which can be as large as any other plan description.
+const SMALL_DESCRIPTION_COMMANDS                      = new Set(
+  [...KEPT_ARGUMENTS_COMMANDS].filter((command) => command !== 'SaveIntoDataSourceCommand'),
+);
+const PLAN_HEADER = '== Physical Plan ==\\nExecute ';
+// The kept form once JSON-parsed: `<Command>\nArguments: <line>`.
+const KEPT_ARGUMENTS = /^[A-Za-z]+\nArguments: /;
 
 // Index of the closing quote of the JSON string whose content starts at `valueStart`: the first
 // quote preceded by an even number of backslashes. -1 when the string is unterminated.
@@ -1326,7 +1371,48 @@ function closingQuoteAt(buf            , limit        , carried        )
   return -1;
 }
 
-// Returns `line` with the physicalPlanDescription string value emptied, or `line` unchanged when
+// The command named by a description that starts at text[valueStart] with PLAN_HEADER, when text
+// holds the whole name; null otherwise.
+function describedCommand(text        , valueStart        )                {
+  if (!text.startsWith(PLAN_HEADER, valueStart)) return null;
+  const nameStart = valueStart + PLAN_HEADER.length;
+  let nameEnd = nameStart;
+  while (nameEnd < text.length && /[A-Za-z]/.test(text[nameEnd])) nameEnd++;
+  return nameEnd < text.length ? text.slice(nameStart, nameEnd) : null;
+}
+
+// What to do with an open description value whose first bytes are text[valueStart...]: 'keep' it
+// (a small Delta command description), 'skip' it, or 'wait' when too little has arrived to tell.
+function descriptionDisposition(text        , valueStart        )                           {
+  const seen = text.slice(valueStart, valueStart + PLAN_HEADER.length);
+  if (!PLAN_HEADER.startsWith(seen)) return 'skip';
+  if (seen.length < PLAN_HEADER.length) return 'wait';
+  const command = describedCommand(text, valueStart);
+  if (command === null) return 'wait'; // the command name is still arriving
+  return SMALL_DESCRIPTION_COMMANDS.has(command) ? 'keep' : 'skip';
+}
+
+// The `<Command>\nArguments: <line>` part (still JSON-escaped) of the root command of a plan
+// description whose value spans line[valueStart, quote), or null when it is not one of
+// KEPT_ARGUMENTS_COMMANDS. SaveIntoDataSourceCommand is kept only for the Delta source: its other
+// providers (JDBC) print option maps that this tool has no use for.
+function keptCommandArguments(line        , valueStart        , quote        )                {
+  const command = describedCommand(line, valueStart);
+  if (command === null || !KEPT_ARGUMENTS_COMMANDS.has(command)) return null;
+  const key = `${command}\\nArguments: `;
+  const at = line.indexOf(key, valueStart + PLAN_HEADER.length + command.length);
+  if (at === -1 || at >= quote) return null;
+  // The line ends at the first escaped newline: a backslash starts a two-char escape.
+  let end = at + key.length;
+  while (end < quote && !(line.charCodeAt(end) === BACKSLASH && line.charCodeAt(end + 1) === LOWER_N)) {
+    end += line.charCodeAt(end) === BACKSLASH ? 2 : 1;
+  }
+  const kept = line.slice(at, Math.min(end, quote));
+  return command === 'SaveIntoDataSourceCommand' && !kept.includes('DeltaDataSource') ? null : kept;
+}
+
+// Returns `line` with the physicalPlanDescription string value emptied (or cut to the kept
+// arguments line, see keptCommandArguments), or `line` unchanged when
 // the key isn't found in Spark's compact form. The key pattern can't match inside another JSON
 // string: there its quotes would be backslash-escaped. buildChunkDecoder already empties a value
 // that spans chunks, so here the value is empty or within one chunk.
@@ -1339,7 +1425,7 @@ export function stripPlanDescription(line        )         {
   const quote = closingQuoteIndex(line, valueStart);
   // Unterminated string (a truncated line): leave it for JSON.parse to reject.
   if (quote === -1) return line;
-  return line.slice(0, valueStart) + line.slice(quote);
+  return line.slice(0, valueStart) + (keptCommandArguments(line, valueStart, quote) ?? '') + line.slice(quote);
 }
 
 const TASK_END_PREFIX = '{"Event":"SparkListenerTaskEnd",';

package/vendor-core/event-schemas.js

@@ -321,13 +321,15 @@ export const TaskEndEventSchema = z.object({
 export const SqlExecutionStartEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart'),
   executionId: z.number(),
+  rootExecutionId: z.number().optional(),
   description: z.string().optional(),
+  physicalPlanDescription: z.string().optional(),
   time: z.number(),
   sparkPlanInfo: SparkPlanInfoFieldSchema,
 });
 
 // applyAdaptiveExecutionUpdate: AQE re-plans mid-query and re-emits sparkPlanInfo for the same
-// executionId; last write wins. Neither SQL schema declares physicalPlanDescription: nothing reads
+// executionId; last write wins. This schema does not declare physicalPlanDescription: nothing reads
 // it, so zod strips it (dispatchLine already empties it before JSON.parse, see stripPlanDescription).
 export const SqlAdaptiveExecutionUpdateEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate'),

package/vendor-core/evidence-report.js

@@ -405,13 +405,13 @@ function verdictJson(model                 )              {
 // A WeakMap needs no invalidation: once mcp-tools.ts evicts the appModel, this entry is collectible.
 // Each cache is split first by the overrides object the report ran under (one fixed, frozen object
 // per CLI invocation or MCP server process; DEFAULT_THRESHOLDS for the specification's).
-                                                                          
+                                                                                 
 const DEFAULT_THRESHOLDS = {};
 const jsonCache              = new WeakMap();
 // The redacted report, keyed by the unredacted appModel it was built from.
 const redactedJsonCache              = new WeakMap();
 
-function cacheFor(cache             , thresholds                                )                                        {
+function cacheFor   (cache                , thresholds                                )                       {
   const key = thresholds ?? DEFAULT_THRESHOLDS;
   let byModel = cache.get(key);
   if (!byModel) {
@@ -421,14 +421,23 @@ function cacheFor(cache             , thresholds
   return byModel;
 }
 
-function runFindings(appModel          , thresholds                                )                                            {
+const findingsCache                                                         = new WeakMap();
+
+/** The run's detector catalog and config audit, computed once per run and overrides so the report
+ * and the blocks built beside it (run-output.ts) share one analysis. */
+export function runFindings(appModel          , thresholds                                )                                            {
+  const cache = cacheFor(findingsCache, thresholds);
+  const cached = cache.get(appModel);
+  if (cached) return cached;
   const { app, stages, executors, sql, jobs, runAggregates } = appModel;
   const catalog = analyze(
     app, stages, executors?.added ?? [], executors?.removed ?? [],
     jobs ?? new Map(), sql ?? new Map(),
     runAggregates ?? null, { thresholds },
   );
-  return { catalog, config: auditConfig(app) };
+  const result = { catalog, config: auditConfig(app) };
+  cache.set(appModel, result);
+  return result;
 }
 
 // Redacts the model and findings before the report derives any text from them, the same order

package/vendor-core/mcp-server-factory.js

@@ -6,6 +6,7 @@ import {
 } from './mcp-tools.js';
 import { listRuns } from './list-runs.js';
                                                          
+import { MAX_NORMALIZE_PATTERN_LENGTH, MAX_NORMALIZE_PATTERNS } from './stage-alignment.js';
 
 const sourceSchema = z.union([
   z.object({ path: z.string() }),
@@ -71,7 +72,7 @@ export function createMcpServer({ thresholds }
   }, (params) => toolResult(listRuns(params)));
 
   server.registerTool('diagnose_run', {
-    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, and the checks the log lacked the data to run. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
+    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, the checks the log lacked the data to run, and the writeTargets, metrics and effectiveConf blocks of the CLI report. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
     inputSchema: {
       ...runRefSchema, redact: z.boolean().optional(),
       include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
@@ -91,15 +92,19 @@ export function createMcpServer({ thresholds }
   ));
 
   server.registerTool('compare_runs', {
-    description: 'Compare two runs: categorized findings delta and metric deltas.',
+    description: 'Compare two runs: the verdict, metric deltas (metrics), categorized findings delta (findings) and the stage pairs (stagePairs) with the share of executor run time they cover (runtimeCoverage). confidence is ok, low or insufficient. metricDeltas and findingsDelta repeat metrics and findings under their earlier names and will be removed.',
     inputSchema: {
       runIdA: z.string().optional(), sourceA: sourceSchema.optional(),
       ...secondRunRefSchema,
       redact: z.boolean().optional(),
+      include: z.array(z.enum(['stagePairs'])).optional()
+        .describe('Extra blocks to return. stagePairs: one row per paired stage with its deltas, which can be large.'),
+      normalizePath: z.array(z.string().max(MAX_NORMALIZE_PATTERN_LENGTH)).max(MAX_NORMALIZE_PATTERNS).optional()
+        .describe('Regular expressions whose every match in a plan node\'s text is replaced with a fixed token before stages are paired, so run-specific text (an output directory that differs per run) does not keep the same stage from pairing. A generic pattern, not only for paths. Affects stage pairing, runtimeCoverage and confidence, never findings. A pattern that backtracks catastrophically can stall the server: keep them simple and anchored.'),
       ...formatSchema,
     },
-  }, ({ runIdA, sourceA, runIdB, sourceB, redact, format }) => toolResultWithMarkdown(
-    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds }),
+  }, ({ runIdA, sourceA, runIdB, sourceB, redact, include, normalizePath, format }) => toolResultWithMarkdown(
+    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds, normalizePath, include }),
   ));
 
   server.registerTool('evaluate_budgets', {
@@ -112,14 +117,16 @@ export function createMcpServer({ thresholds }
       sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced). When given, the absolute budgets (maxRuntimeMs etc.) are evaluated on this run.'),
       runIdB: secondRunRefSchema.runIdB.describe('Same as `sourceB`, referencing an already-resolved run by id.'),
       maxRegressionPct: z.number().optional(), regressionMetric: z.string().optional(), failOnIntroduced: z.string().optional(),
+      regressionBudgets: z.array(z.object({ metric: z.string(), maxPct: z.number().nonnegative() })).optional()
+        .describe('Further regression budgets, one per metric (the CLI\'s repeated --regression-budget). A metric can be budgeted once across this list and maxRegressionPct/regressionMetric.'),
     },
   }, ({
     source, runId, runIdB, sourceB,
     maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct,
-    maxRegressionPct, regressionMetric, failOnIntroduced,
+    maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets,
   }) => toolResult(evaluateBudgetsForRun(
     { source, runId },
-    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced },
+    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets },
     (runIdB || sourceB) ? { runId: runIdB, source: sourceB } : undefined,
     { thresholds },
   )));