sparkforensics-mcp 0.4.0 → 0.5.0

package/vendor-core/recommendation-rollup.js

@@ -1,7 +1,7 @@
 // Explicit .ts extensions: plain Node's ESM resolver (the runtime CLI/MCP path
 // runs under) requires the exact specifier, unlike a bundler.
 import { mergeIntervals } from './intervals.js';
-import { formatWallClockRange, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
+import { formatRawWaste, formatWallClockRange, readsAsZero, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
                                                                     
 
                          
@@ -124,11 +124,15 @@ export function buildRecommendationRollup(
   });
 }
 
-/** A group's trailing figure on the Findings board, and its tooltip. `time`
- * and `resource` both lead with the "×N" finding count (the "worth
- * expanding" signal); `count` skips it since the impact-band tally already
- * implies N. The row stays terse ("×2 · 476ms recoverable") to fit a dense
- * right-aligned column; the title spells the shorthand out. */
+/** A resource group's summed waste as the board and CLI print it; null when it reads as zero. */
+export function resourceGroupTotal(group                                       )                {
+  const text = formatRawWaste({ value: group.total, unit: group.unit });
+  return readsAsZero(text) ? null : text;
+}
+
+/** A group's trailing figure on the Findings board, and its tooltip. Every kind leads with the
+ * "×N" finding count (the "worth expanding" signal). The row stays terse ("×2 · 476ms
+ * recoverable") to fit a dense right-aligned column; the title spells the shorthand out. */
 export function rollupGroupStat(group             )                                      {
   if (group.kind === 'time') {
     const recoverable = formatWallClockRange(group.recoverableMsHigh, group.recoverableMsHigh);
@@ -138,13 +142,16 @@ export function rollupGroupStat(group             )
     };
   }
   if (group.kind === 'resource') {
+    const total = resourceGroupTotal(group);
     return {
-      stat: `×${group.findingCount} · resource-cost projection`,
-      statTitle: `${group.findingCount} findings of this type; a resource-cost estimate (not run time) is projected for fixing them`,
+      stat: total ? `×${group.findingCount} · ${total}` : `×${group.findingCount}`,
+      statTitle: `${group.findingCount} findings of this type${total ? `; ${total} in total, a resource cost, not run time` : ''}`,
     };
   }
+  // The band heading above the row already names a one-band group's band.
+  const bands = Object.entries(group.byImpactBand);
   return {
-    stat: Object.entries(group.byImpactBand).map(([impactBand, count]) => `${count} ${impactBand}`).join(', '),
+    stat: bands.length === 1 ? `×${group.findingCount}` : `×${group.findingCount} · ${bands.map(([impactBand, count]) => `${count} ${impactBand}`).join(', ')}`,
     statTitle: `${group.findingCount} findings of this type, by impact`,
   };
 }

package/vendor-core/redact.js

@@ -9,13 +9,12 @@
 
 import { decodeCollections, encodeCollections,                    } from './export-data.js';
                                                     
-import { redactTaskFailureGroup,                        } from './task-failure.js';
+import { REDACTED_TEXT, redactTaskFailureGroup,                        } from './task-failure.js';
 
 // Host / IP identifier patterns. Used to enumerate host names that surface only
-// inside free text: recommendation strings, `stageFailed`'s failure-reason
-// value, SQL relation/node names, never as a structured `host` field, so
-// redaction reaches those residuals too. Pseudonyms (`host-1`) match neither
-// pattern, keeping the scan idempotent.
+// inside free text: recommendation strings, SQL relation/node names, never as a
+// structured `host` field, so redaction reaches those residuals too. Pseudonyms
+// (`host-1`) match neither pattern, keeping the scan idempotent.
 const HOST_PATTERNS = [
   // EC2-style ip-10-1-2-3 with an optional dotted domain (ip-10-1-2-3.ec2.internal).
   // Each domain label must start with an alphanumeric, so a trailing sentence
@@ -95,6 +94,31 @@ function redactFailureGroups   (node   )    {
   return node;
 }
 
+// A stage's failure reason is Spark's free-text message and can carry file paths and data values
+// too, so it is replaced outright, like a failure group's message: a `stageFailed` finding's
+// `valueText`, the `stageFailureReason` of a stage record, a job's `exception` (the run outcome's
+// fallback reason), and the report's `failureReason` copy. Walking by key name, like
+// collectHostFields, needs no path list. Returns a fresh tree.
+const REASON_KEYS = new Set(['stageFailureReason', 'exception', 'failureReason']);
+function redactStageFailureReasons   (node   )    {
+  if (Array.isArray(node)) return node.map((n) => redactStageFailureReasons(n))     ;
+  if (node && typeof node === 'object') {
+    const isStageFailed = (node                      ).type === 'stageFailed';
+    const out                          = {};
+    for (const [k, v] of Object.entries(node)) {
+      const isReason = REASON_KEYS.has(k) || (isStageFailed && k === 'valueText');
+      out[k] = isReason && typeof v === 'string' ? REDACTED_TEXT : redactStageFailureReasons(v);
+    }
+    return out     ;
+  }
+  return node;
+}
+
+// Both passes for free text no host/app-id pattern recognizes.
+function redactFailureText   (node   )    {
+  return redactStageFailureReasons(redactFailureGroups(node));
+}
+
 // Spark config keys ending in `host`/`hostname` (e.g. spark.driver.host,
 // spark.yarn.am.hostname) carry plain FQDN host names that neither
 // HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
@@ -122,7 +146,7 @@ function collectConfigHostValues(config                                    , hos
                             
                                                                           
                                                                               
-                                             
+                                                                   
                                                                                
                                                                               
                  
@@ -188,10 +212,19 @@ function applyReplacements   (node   , ids
   return deepReplace(node, merged)     ;
 }
 
+// The app name identifies a job as much as its id, so a redacted name becomes the id's
+// pseudonym, as list_runs does. Only the structured field is replaced, never substrings: a
+// short name ("t", "etl") would otherwise corrupt every string it happens to occur in.
+function withRedactedName                                                        (app   )    {
+  return app.name == null ? app : { ...app, name: app.id ?? null };
+}
+
 export function redactReport                            (input   )    {
-  const report = redactFailureGroups(input);
+  const report = redactFailureText(input);
   const { appIds, hosts } = collectIds(report);
-  return applyReplacements(report, { appIds, hosts });
+  const out = applyReplacements(report, { appIds, hosts });
+  const app = out.summary?.app;
+  return app ? { ...out, summary: { ...out.summary, app: withRedactedName(app) } } : out;
 }
 
 // Run-comparison counterpart: no single app-id *field* to pseudonymize
@@ -218,7 +251,7 @@ export function redactComparison   (comparison   )    {
                                                                                        
 
 function redactRunTree                   (input   )    {
-  const data = redactFailureGroups(input);
+  const data = redactFailureText(input);
   const appIds = new Set        ();
   const hosts = new Set        ();
   const appId = data.app?.id;
@@ -229,7 +262,12 @@ function redactRunTree                   (input   )    {
   collectHostFields(data.configFindings, hosts);
   collectConfigHostValues(data.app?.config, hosts);
   scanTokens(data, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
-  return applyReplacements(data, { appIds, hosts });
+  const out = applyReplacements(data, { appIds, hosts });
+  if (!out.app) return out;
+  const app = withRedactedName(out.app);
+  // spark.app.name repeats the name in the config the HTML export ships.
+  if (app.config?.['spark.app.name'] == null) return { ...out, app };
+  return { ...out, app: { ...app, config: { ...app.config, 'spark.app.name': app.id ?? '' } } };
 }
 
 /** A run's model and findings with every identifier pseudonymized. Redact this before anything derives
@@ -264,6 +302,9 @@ export function redactRunModel(
       executors: redacted.executors,
       runAggregates: redacted.runAggregates                             ,
       evidenceAvailability: redacted.evidenceAvailability                                    ,
+      // Counts and execution ids, nothing to pseudonymize.
+      skippedLines: appModel.skippedLines,
+      unreadableSqlExecutions: appModel.unreadableSqlExecutions,
     }            ,
     catalog: redacted.catalog,
     configFindings: redacted.configFindings,

package/vendor-core/remediation.js

@@ -0,0 +1,20 @@
+// Constructors for the structured `remediation` a detector attaches next to its prose
+// `recommendation`: only where the detector already names the Spark property, never an invented one.
+                                                      
+
+                                          
+
+/** The property should be raised; `suggested` is the value when the detector computed one. */
+export function increaseConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'increase', suggested };
+}
+
+/** The property should be lowered; `suggested` is the value when the detector computed one. */
+export function decreaseConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'decrease', suggested };
+}
+
+/** The property should take a specific value (a switch or a class name), not move up or down. */
+export function setConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'set', suggested };
+}

package/vendor-core/run-comparison.js

@@ -1,6 +1,9 @@
 import { computeWallClock } from './wall-clock.js';
 import { normalizeDetail } from './detectors.js';
 import { cyrb53 } from './string-hash.js';
+import { computeAllocation } from './allocation.js';
+import { totalExecutorCpuMs, withEarlierAttempts } from './run-totals.js';
+import { planNodesOfStage } from './stage-plan-nodes.js';
 import { captureSnapshot } from './session-snapshot.js';
 import { tunedRunNote } from './threshold-overrides.js';
                                                                                                     
@@ -76,23 +79,18 @@ function planTreeIdentity(root                             )                {
 // Falls back to the coarser whole-tree identity when the stage has no
 // attributed nodes (hand-built snapshots without `stageIds`, or unmatched
 // accumulables).
-function sqlNodeIdentity(stage       , snapshot                 )         {
+function sqlNodeIdentity(stage       , snapshot                              )         {
   const execId = stage.sqlExecutionId;
   if (execId == null) return '';
   const root = snapshot.sql.get(execId)?.planTree ?? null;
   if (!root) return '';
-  const fingerprints           = [];
-  (function collect(node          )       {
-    if (node.stageIds?.includes(stage.id)) {
-      fingerprints.push(JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
-    }
-    for (const child of node.children ?? []) collect(child);
-  })(root);
+  const fingerprints = planNodesOfStage(stage, snapshot.sql)
+    .map((node) => JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
   if (fingerprints.length === 0) return planTreeIdentity(root) ?? '';
   return cyrb53(JSON.stringify(fingerprints.sort()));
 }
 
-export function stageIdentity(stage       , snapshot                 )         {
+export function stageIdentity(stage       , snapshot                              )         {
   return normalizeStageName(stage.name ?? '') + '§' + sqlNodeIdentity(stage, snapshot);
 }
 
@@ -199,6 +197,7 @@ function skewRatios(stages         )           {
 export const COMPARISON_METRIC_KEYS                    = [
   'wallClock', 'shuffleSpill', 'taskSkew', 'failedTaskRate', 'diskSpill', 'gcTime',
   'inputBytes', 'outputBytes', 'executorRunTime', 'taskCount', 'executorsAdded',
+  'executorCpuTime', 'allocatedCoreHours',
 ];
 
 // Volume/count metrics, not cost metrics: more or less input/output data, or
@@ -228,20 +227,24 @@ function metric(
 
 export function metricDeltas(baseSnap                 , candSnap                 )                   {
   const out                   = [];
+  // Run totals count every task attempt, failed and speculative ones too (run-totals.ts), the
+  // same sums the CLI metrics block reports; skew stays on each stage's latest attempt.
+  const attemptsOf = (snap                 )          => withEarlierAttempts([...snap.stages.values()]);
 
   // Wall-clock: always computable (computeWallClock tolerates a null app).
   out.push(metric('wallClock', 'Wall-clock duration',
     computeWallClock(baseSnap.app, baseSnap.stages).total,
     computeWallClock(candSnap.app, candSnap.stages).total));
 
-  // Shuffle spill over the whole run: a sum needs no stage matching, and
-  // matching is unreliable on real logs (see plan rationale), so scope it to
-  // all stages exactly like task-skew and failed-rate below.
-  const bSpill = sumField([...baseSnap.stages.values()], 'memoryBytesSpilled');
-  const cSpill = sumField([...candSnap.stages.values()], 'memoryBytesSpilled');
-  out.push(metric('shuffleSpill', 'Shuffle spill',
+  // Memory spill (Spark's memoryBytesSpilled) over the whole run: a sum needs no
+  // stage matching, and matching is unreliable on real logs, so scope it to all
+  // stages exactly like task-skew and failed-rate below. The key stays
+  // `shuffleSpill` so existing --regression-metric callers keep working.
+  const bSpill = sumField(attemptsOf(baseSnap), 'memoryBytesSpilled');
+  const cSpill = sumField(attemptsOf(candSnap), 'memoryBytesSpilled');
+  out.push(metric('shuffleSpill', 'Memory spill',
     bSpill.present ? bSpill.sum : null, cSpill.present ? cSpill.sum : null,
-    { unavailableReason: bSpill.present && cSpill.present ? undefined : 'No shuffle-spill data recorded for a run' }));
+    { unavailableReason: bSpill.present && cSpill.present ? undefined : 'No memory-spill data recorded for a run' }));
 
   // Task skew: p95 of per-stage ratio across the whole run.
   const bSkew = p95(skewRatios([...baseSnap.stages.values()]));
@@ -250,10 +253,10 @@ export function metricDeltas(baseSnap                 , candSnap
     { unavailableReason: bSkew != null && cSkew != null ? undefined : 'No stage had measurable duration for a run' }));
 
   // Failed-task rate: Σ failedTasks / Σ taskCount.
-  const bTasks = sumField([...baseSnap.stages.values()], 'taskCount');
-  const cTasks = sumField([...candSnap.stages.values()], 'taskCount');
-  const bFailed = sumField([...baseSnap.stages.values()], 'failedTasks');
-  const cFailed = sumField([...candSnap.stages.values()], 'failedTasks');
+  const bTasks = sumField(attemptsOf(baseSnap), 'taskCount');
+  const cTasks = sumField(attemptsOf(candSnap), 'taskCount');
+  const bFailed = sumField(attemptsOf(baseSnap), 'failedTasks');
+  const cFailed = sumField(attemptsOf(candSnap), 'failedTasks');
   // Guard on BOTH inputs: a missing `failedTasks` field must render Unavailable,
   // not a false 0% rate (dividing an absent-and-therefore-0 numerator).
   const bRate = bTasks.present && bTasks.sum > 0 && bFailed.present ? bFailed.sum / bTasks.sum : null;
@@ -265,8 +268,8 @@ export function metricDeltas(baseSnap                 , candSnap
   // carries (set in finalizeStage). Correct at any match coverage, like the
   // sums above; no parser or detector change.
   const sumMetric = (key        , label        , field                   , reason        ) => {
-    const b = sumField([...baseSnap.stages.values()], field);
-    const c = sumField([...candSnap.stages.values()], field);
+    const b = sumField(attemptsOf(baseSnap), field);
+    const c = sumField(attemptsOf(candSnap), field);
     out.push(metric(key, label, b.present ? b.sum : null, c.present ? c.sum : null,
       { unavailableReason: b.present && c.present ? undefined : reason }));
   };
@@ -285,6 +288,18 @@ export function metricDeltas(baseSnap                 , candSnap
   out.push(metric('executorsAdded', 'Executors added', bExec, cExec,
     { unavailableReason: bExec != null && cExec != null ? undefined : 'No executor events recorded for a run' }));
 
+  // Executor CPU time (ms) and allocated core-hours cost resources, so less is better. CPU time is
+  // null, not 0, on a run whose log never recorded it (older Spark).
+  const cpuMs = (snap                 )                => totalExecutorCpuMs(attemptsOf(snap));
+  const bCpu = cpuMs(baseSnap), cCpu = cpuMs(candSnap);
+  out.push(metric('executorCpuTime', 'Executor CPU time', bCpu, cCpu,
+    { unavailableReason: bCpu != null && cCpu != null ? undefined : 'No executor CPU time recorded for a run' }));
+  const coreHours = (snap                 )                =>
+    (snap.executors ? computeAllocation(snap).coreHours : null);
+  const bCore = coreHours(baseSnap), cCore = coreHours(candSnap);
+  out.push(metric('allocatedCoreHours', 'Allocated core-hours', bCore, cCore,
+    { unavailableReason: bCore != null && cCore != null ? undefined : 'No executor lifecycle or core count recorded for a run' }));
+
   return out;
 }
 
@@ -467,8 +482,12 @@ export function renderComparisonMarkdown(
   const lines = ['', '## Comparison to baseline', ''];
   if (tuned) lines.push(`- Tuned thresholds (both runs): ${tunedRunNote(tuned)}`, '');
   if (verdict) {
-    // The dashboard comparison page's headline: run A is the baseline, run B the candidate.
-    lines.push(`Run A: ${comparison.baselineLabel} · Run B: ${comparison.candidateLabel}`, '', verdict.title);
+    // Names each run by its label (MCP's run IDs), unless the labels are just the role names the
+    // CLI passes, where the verdict below already says baseline and candidate.
+    if (comparison.baselineLabel !== 'baseline' || comparison.candidateLabel !== 'candidate') {
+      lines.push(`Baseline: ${comparison.baselineLabel} · Candidate: ${comparison.candidateLabel}`, '');
+    }
+    lines.push(verdict.title);
     if (verdict.sentences.length > 0) lines.push('', verdict.sentences.join(' '));
     lines.push('');
   }

package/vendor-core/run-interpretation.js

@@ -192,6 +192,7 @@ function interpretEfficiency(appModel          )                        {
     app: appModel.app,
     stages: appModel.stages,
     executorsAdded: appModel.executors.added,
+    executorsRemoved: appModel.executors.removed,
     runAggregates: appModel.runAggregates,
   });
 }
@@ -270,7 +271,7 @@ export function interpretRun(appModel          , catalog           , configFindi
     coverage: interpretCoverage(appModel, allFindings),
     runShape: interpretRunShape(appModel),
     wallClock: computeWallClock(appModel.app, appModel.stages),
-    wastedCoreHours: computeWastedCoreHours(appModel.app, appModel.executors.added, appModel.runAggregates),
+    wastedCoreHours: computeWastedCoreHours(appModel.app, appModel.executors.added, appModel.runAggregates, appModel.executors.removed),
     efficiency: interpretEfficiency(appModel),
     wallClockReliable: checkConcurrentJobGroups(appModel.jobs).wallClockReliable,
     etlPhases: attributeEtlPhases(appModel.stages),

package/vendor-core/run-metrics.js

@@ -0,0 +1,198 @@
+// The machine-readable metrics block of the CLI's JSON output: run-level totals and per-stage rows
+// an automated tuning loop can compare across runs without reading a report. Its schemaVersion
+// moves independently of the evidence report's. A figure the log cannot provide is null, never 0.
+import { computeAllocation,                 } from './allocation.js';
+import { computeSkewRatio, ENTRY_BY_TYPE,                         } from './detectors.js';
+import { totalExecutorCpuMs, withEarlierAttempts } from './run-totals.js';
+import { isPythonStage } from './python-stage.js';
+import { stageIdentity } from './run-comparison.js';
+import { hasCompleteApplicationInterval } from './scorecard-estimates.js';
+import { effectiveThresholds } from './threshold-overrides.js';
+import { computeWallClock } from './wall-clock.js';
+                                                  
+
+export const METRICS_SCHEMA_VERSION = 1;
+
+                               
+                            
+                                   
+                                   
+                          
+                                  
+                                
+                                  
+                                   
+                            
+                             
+                            
+                                          
+                           
+                             
+                              
+                      
+ 
+
+                                                       
+                                                                                                  
+                     
+                                                                                               
+                                 
+                         
+                          
+                  
+ 
+
+                             
+                        
+                                                                                                  
+                                                                                               
+                       
+                                                                                                                                    
+         
+                                                                   
+                                                                      
+                                                                                     
+                                            
+    
+          
+                                                 
+                                                                                                 
+                                                                                       
+                                                                     
+                                                                                    
+    
+                         
+           
+                                                                                             
+                                      
+    
+                                          
+ 
+
+const finite = (v         )              => typeof v === 'number' && Number.isFinite(v);
+
+// Σ of a field over the stages that carry a finite value; null when none does.
+function sumOf(stages         , pick                                         )                {
+  let sum = 0, present = false;
+  for (const s of stages) {
+    const v = pick(s);
+    if (finite(v)) { sum += v; present = true; }
+  }
+  return present ? sum : null;
+}
+
+// 0 means either no execution memory used or not recorded; only a positive peak is reported.
+function peakExecutionMemory(stages         )                {
+  const peaks = stages.map((s) => s.peakExecutionMemoryMax).filter((v)              => finite(v) && v > 0);
+  return peaks.length > 0 ? Math.max(...peaks) : null;
+}
+
+function maxSkew(stages         , minTasksForP95        )                {
+  const ratios = stages
+    .map((s) => computeSkewRatio(s, minTasksForP95)?.ratio)
+    .filter((r)              => finite(r));
+  return ratios.length > 0 ? Math.max(...ratios) : null;
+}
+
+// Failed stage attempts and stages submitted more than once, from the parser's per-stage attempt
+// counts; null when any stage record lacks them.
+function stageAttempts(stages         )                                             {
+  let failed = 0, retried = 0;
+  for (const s of stages) {
+    if (!finite(s.stageAttempts) || !finite(s.failedStageAttempts)) return null;
+    failed += s.failedStageAttempts;
+    if (s.stageAttempts > 1) retried++;
+  }
+  return { failed, retried };
+}
+
+// Metrics of a set of stages, every attempt included; a row folds the stages sharing one
+// fingerprint. Skew describes the latest attempt's task durations.
+function stageMetrics(stages         , minTasksForP95        )               {
+  const attempts = withEarlierAttempts(stages);
+  // Task-derived figures are null for stages that never finished (no task records).
+  // Superseded attempts carry work but no task count of their own, so they stay in once any
+  // attempt finished a task.
+  const finished = attempts.some((s) => (s.taskCount ?? 0) > 0) ? attempts : [];
+  const tasks = sumOf(attempts, (s) => s.taskCount);
+  const fromTasks = (pick                                  )                => (finished.length > 0 ? sumOf(finished, pick) : null);
+  const durations = attempts.filter((s) => finite(s.submittedAt) && finite(s.completedAt) && (s.completedAt          ) >= (s.submittedAt          ));
+  return {
+    durationMs: durations.length > 0 ? sumOf(durations, (s) => (s.completedAt          ) - (s.submittedAt          )) : null,
+    executorCpuTimeMs: totalExecutorCpuMs(finished),
+    executorRunTimeMs: fromTasks((s) => s.executorRunTime),
+    gcTimeMs: fromTasks((s) => s.jvmGCTime),
+    memorySpillBytes: fromTasks((s) => s.memoryBytesSpilled),
+    diskSpillBytes: fromTasks((s) => s.diskBytesSpilled),
+    shuffleReadBytes: fromTasks((s) => s.shuffleReadBytes),
+    shuffleWriteBytes: fromTasks((s) => s.shuffleWriteBytes),
+    inputBytes: fromTasks((s) => s.inputBytes),
+    outputBytes: fromTasks((s) => s.outputBytes),
+    outputRows: sumOf(finished, (s) => s.outputRecords),
+    peakExecutionMemoryBytes: peakExecutionMemory(finished),
+    taskCount: tasks != null && tasks > 0 ? tasks : null,
+    failedTasks: fromTasks((s) => s.failedTasks),
+    retriedTasks: fromTasks((s) => (s.wastedAttempts                      ) ?? 0),
+    skew: maxSkew(stages.filter((s) => (s.taskCount ?? 0) > 0), minTasksForP95),
+  };
+}
+
+/** The run's metrics block. `thresholds` only moves the skew ratio's P95 cutoff, as for the
+ * --max-skew budget. */
+export function computeRunMetrics(appModel          , thresholds                     )             {
+  const stageList = [...appModel.stages.values()];
+  const minTasksForP95 = effectiveThresholds(ENTRY_BY_TYPE.get('skew') , thresholds).minTasksForP95          ;
+  const all = stageMetrics(stageList, minTasksForP95);
+  const python = stageList.filter((s) => isPythonStage(s, appModel.sql));
+  const pythonRunTimeMs = sumOf(withEarlierAttempts(python), (s) => s.executorRunTime);
+
+  const byFingerprint = new Map                 ();
+  for (const s of stageList) {
+    const key = stageIdentity(s, appModel);
+    const group = byFingerprint.get(key);
+    if (group) group.push(s); else byFingerprint.set(key, [s]);
+  }
+  const stages                                  = {};
+  for (const [key, group] of byFingerprint) {
+    const attempts = stageAttempts(group);
+    stages[key] = {
+      stageIds: group.map((s) => s.id).sort((a, b) => a - b),
+      failed: attempts ? attempts.failed > 0 : null,
+      retried: attempts ? attempts.retried > 0 : null,
+      python: group.some((s) => isPythonStage(s, appModel.sql)),
+      ...stageMetrics(group, minTasksForP95),
+    };
+  }
+
+  const attempts = stageAttempts(stageList);
+  return {
+    schemaVersion: METRICS_SCHEMA_VERSION,
+    runComplete: appModel.app?.endTime != null,
+    time: {
+      wallClockMs: hasCompleteApplicationInterval(appModel.app) ? computeWallClock(appModel.app, appModel.stages).total : null,
+      executorCpuTimeMs: all.executorCpuTimeMs,
+      executorRunTimeMs: all.executorRunTimeMs,
+      gcTimeMs: all.gcTimeMs,
+    },
+    data: {
+      memorySpillBytes: all.memorySpillBytes, diskSpillBytes: all.diskSpillBytes,
+      shuffleReadBytes: all.shuffleReadBytes, shuffleWriteBytes: all.shuffleWriteBytes,
+      inputBytes: all.inputBytes, outputBytes: all.outputBytes, outputRows: all.outputRows,
+      peakExecutionMemoryBytes: all.peakExecutionMemoryBytes,
+    },
+    shape: {
+      taskCount: all.taskCount,
+      stageCount: stageList.length,
+      failedStageAttempts: attempts?.failed ?? null,
+      retriedStages: attempts?.retried ?? null,
+      failedTasks: all.failedTasks,
+      retriedTasks: all.retriedTasks,
+      maxSkew: all.skew,
+    },
+    allocation: computeAllocation(appModel),
+    python: {
+      shareOfTaskRunTime: all.executorRunTimeMs != null && all.executorRunTimeMs > 0 ? (pythonRunTimeMs ?? 0) / all.executorRunTimeMs : null,
+    },
+    stages,
+  };
+}

package/vendor-core/run-totals.js

@@ -0,0 +1,24 @@
+// The run-level sums every surface reads: the CLI metrics block and the run comparison (so the
+// dashboard's comparison view and the MCP compare_runs tool) share these, so one name never means
+// two figures.
+import { nsToMs } from './format-utils.js';
+                                        
+
+/** Each stage followed by the work its own figures leave out, shaped like a stage: the attempts a
+ * resubmit replaced, and task attempts the stage record dropped (a failed attempt's late tasks,
+ * failed retries, losing speculative copies). Summing the result counts every attempt's work,
+ * failed ones too. A stage's own record, which the detectors and per-stage views read, is left as
+ * it is. */
+export function withEarlierAttempts(stages         )          {
+  return stages.flatMap((s) => [s, ...[s.earlierAttempts, s.lateAttemptWork]
+    .filter((work) => work != null)
+    .map(({ durationMs, ...totals }) => ({ id: s.id, ...totals, submittedAt: 0, completedAt: durationMs ?? undefined }))]);
+}
+
+/** Summed executor CPU time in ms. Spark records it in nanoseconds and the parser reads an absent
+ * metric as 0, so stages that all read 0 never recorded it (older Spark): null, not 0. */
+export function totalExecutorCpuMs(stages                                          )                {
+  let ns = 0;
+  for (const s of stages) if (typeof s.executorCpuTime === 'number' && Number.isFinite(s.executorCpuTime) && s.executorCpuTime > 0) ns += s.executorCpuTime;
+  return ns > 0 ? nsToMs(ns) : null;
+}

package/vendor-core/run-verdict.js

@@ -236,13 +236,12 @@ export function verdictSummary(eligible           , steps            , facts
   } else {
     sentences.push(`${plural(eligible.length, 'finding')} in ${plural(steps.length, 'place')}.`);
     const wallClock = steps[0].lead.impactEstimate?.wallClock;
-    if (isIdleCapacityStep(steps[0])) {
-      sentences.push('A smaller cluster or dynamic allocation would free the idle cores for other jobs.');
-    } else if (wallClock && facts.runMs != null) {
+    // An idle-capacity lead needs no sentence here: the title gives the idle share and step 1 the fix.
+    if (!isIdleCapacityStep(steps[0]) && wallClock && facts.runMs != null) {
       sentences.push(`The first fix could save up to ${formatDuration(wallClock.high)} of this ${formatDuration(facts.runMs)} run.`);
     }
     if (steps.some((step) => step.related.length > 0)) {
-      sentences.push('Findings in the same stage usually share one cause, so they are grouped together and their savings overlap rather than add up.');
+      sentences.push('Findings on one stage are grouped, and their savings overlap.');
     }
   }
   if (facts.incomplete) {

package/vendor-core/scorecard-estimates.js

@@ -45,6 +45,7 @@ export function getScorecardEstimates(appModel          )
     app: appModel.app,
     stages: appModel.stages,
     executorsAdded: appModel.executors.added,
+    executorsRemoved: appModel.executors.removed,
     runAggregates: appModel.runAggregates,
   });
 

package/vendor-core/session-snapshot.js

@@ -24,6 +24,8 @@ import { isSupportedEvidenceAvailability } from './evidence-availability.js';
                          
                                       
                                                     
+                        
+                                     
                      
                                  
  
@@ -44,6 +46,8 @@ export function captureSnapshot(
     jobs: new Map(appModel.jobs),
     runAggregates: appModel.runAggregates,
     evidenceAvailability: appModel.evidenceAvailability,
+    skippedLines: appModel.skippedLines,
+    unreadableSqlExecutions: appModel.unreadableSqlExecutions && [...appModel.unreadableSqlExecutions],
     catalog: [...catalog],
     taskData: new Map(taskDataCache),
   };
@@ -71,6 +75,9 @@ export function applySnapshot(
   appModel.evidenceAvailability = isSupportedEvidenceAvailability(snapshot.evidenceAvailability)
     ? snapshot.evidenceAvailability
     : null;
+  appModel.skippedLines = snapshot.skippedLines;
+  if (snapshot.unreadableSqlExecutions) appModel.unreadableSqlExecutions = [...snapshot.unreadableSqlExecutions];
+  else delete appModel.unreadableSqlExecutions;
 
   taskDataCache.clear();
   for (const [k, v] of snapshot.taskData) taskDataCache.set(k, v);

package/vendor-core/shs-schemas.js

@@ -1,8 +1,8 @@
 import { z } from 'zod';
 
 // Proxy-level error envelope: the JSON body the local server's SHS proxy
-// (./shs-request.js) sends back on a non-OK upstream response, e.g.
-// `{ code: 'shs-unreachable' }`. This is NOT a SparkListener* event shape, so
+// (./proxy.js) sends back on a non-OK upstream response, e.g.
+// `{ code: 'upstream-unreachable' }`. This is NOT a SparkListener* event shape, so
 // it lives here rather than in event-schemas.ts. `.passthrough()` since the
 // proxy may attach extra debugging fields the consumer doesn't care about;
 // only `code` is read.

package/vendor-core/spark-memory.js

@@ -0,0 +1,17 @@
+// Parse a Spark memory-size string to MiB. Spark's JVM-memory configs use bytesConf(ByteUnit.MiB),
+// so a bare number means MiB. A k/m/g/t suffix sets the unit (trailing "b" redundant); a lone "b"
+// ("10b") means bytes.
+export function parseSparkMemoryMB(value         )                {
+  if (value == null) return null;
+  const m = String(value).trim().toLowerCase().match(/^([\d.]+)\s*([kmgt]?)(b?)$/);
+  if (!m) return null;
+  const n = parseFloat(m[1]);
+  if (!Number.isFinite(n)) return null;
+  switch (m[2]) {
+    case 'k': return Math.round(n / 1024);
+    case 'g': return Math.round(n * 1024);
+    case 't': return Math.round(n * 1024 * 1024);
+    case 'm': return Math.round(n);
+    default: return m[3] === 'b' ? Math.round(n / (1024 * 1024)) : Math.round(n);
+  }
+}