sparkforensics-server 0.5.2 → 0.8.1

package/vendor-core/run-comparison.js

@@ -6,41 +6,79 @@ import { totalExecutorCpuMs, withEarlierAttempts } from './run-totals.js';
 import { captureSnapshot } from './session-snapshot.js';
 import { alignStages, compileNormalizePatterns, COMPARISON_SCHEMA_VERSION,                                     } from './stage-alignment.js';
 import { tunedRunNote } from './threshold-overrides.js';
-                                                                                          
+import { diffFindings, findingBand, findingKey, findingRule,                     } from './finding-identity.js';
+import { cyrb53 } from './string-hash.js';
+import { IMPACT_BAND_ORDER } from './format-utils.js';
+                                                                                                      
                                                              
                                                                                         
 import { isIncompleteRun } from './check-coverage.js';
+import { stageDisplayLabel } from './stage-label.js';
+import { runAppIdentity,                     } from './run-identity.js';
 import { summarizeRunOutcome } from './run-outcome.js';
+import { buildComparability, withinBand,                    } from './comparability.js';
 
                                  
                              
                                                                           
                                                                                     
                              
+                                                                                                 
+                                                
+                       
  
+// An introduced or resolved finding category: the findings of one rule and band that have no
+// counterpart in the other run. An introduced row has `baseCount` 0, a resolved row `candCount` 0.
+// `keys`: the `key` of each finding the row counts, as `diagnose_run` reports it for the run the
+// findings are in (the candidate for an introduced row, the baseline for a resolved one).
                                    
                                                                                                                         
+                                                                                                  
+                        
+                 
+ 
+// Findings both runs have, at one location, that moved from one impact band to another. `count`
+// is how many findings of that rule moved between those two bands.
+// `keys` are the candidate run's finding keys, `baseKeys` the baseline's: a pair the aligner matched
+// without an exact fingerprint match has different keys on the two sides.
+                                     
+                                                                                                              
+                                                                                                  
+                        
+                                     
+ 
+/** True when a changed finding moved to a more severe band (an unknown band ranks least severe). */
+export function isEscalation(row                                                               )          {
+  const rank = (band        ) => IMPACT_BAND_ORDER[band              ] ?? 9;
+  return rank(row.candImpactBand) < rank(row.baseImpactBand);
+}
+                                
+                                                                                              
  
                                
                                                                
                                                                           
  
+/** A stage pair with its stage name as a display label (call-site directories removed). The pair
+ * carries no raw name, so the label is the only name on the row. */
+                                                                          
                                     
                                                 
+                                                            
                                                                                                 
                                                                    
-                                                                                                 
+                                                                                             
                                                                          
                           
                                                             
                                  
-                          
+                                 
                                          
                                          
                                                              
                                                            
                             
-                                                                             
+                          
                             
                                                                             
                                                                             
@@ -48,6 +86,7 @@ import { summarizeRunOutcome } from './run-outcome.js';
                                                                                                
                                    
                                                                              
+                               
  
 
 function jobOutcome(snapshot                 )                    {
@@ -61,6 +100,41 @@ export function stageIdentity(stage       , snapshot
   return stageIdentityWith(stage, snapshot, normalizeDetail);
 }
 
+/** The key a finding row carries (`findingKey`: type, rule and location) with the location read off
+ * the run alone: a stage by its `stageIdentity` fingerprint (the keys of `metrics.stages`), an SQL
+ * execution by a digest of its stages' fingerprints. Two runs of the same job give the same finding
+ * the same key, so a reader pairs findings across runs without the aligner. A pairing aid: several
+ * findings can share a key (one detector firing per host on a run-level key, stages that share a
+ * fingerprint), and it says nothing about whether the finding's value or band held. */
+export function findingKeyer(snapshot                                         )                               {
+  const stageLabels = new Map                ();
+  const stageLabel = (id        )         => {
+    let label = stageLabels.get(id);
+    if (label === undefined) {
+      const stage = snapshot.stages.get(id);
+      label = stage ? stageIdentity(stage, snapshot) : `#${id}`;
+      stageLabels.set(id, label);
+    }
+    return label;
+  };
+  let stagesByExecution                                   ;
+  const executionLabel = (id        )         => {
+    if (!stagesByExecution) {
+      stagesByExecution = new Map();
+      for (const [stageId, stage] of snapshot.stages) {
+        if (stage.sqlExecutionId == null) continue;
+        const ids = stagesByExecution.get(stage.sqlExecutionId);
+        if (ids) ids.push(stageId); else stagesByExecution.set(stage.sqlExecutionId, [stageId]);
+      }
+    }
+    const ids = stagesByExecution.get(id);
+    // An execution with no stage has no fingerprint to digest: its id is all that names it.
+    return ids ? cyrb53(JSON.stringify(ids.map(stageLabel).sort())) : `#${id}`;
+  };
+  const locator                 = { stage: stageLabel, execution: executionLabel };
+  return (finding) => findingKey(finding, locator);
+}
+
 export function matchStages(baseSnap                 , candSnap                 )   
                                                                      
                                  
@@ -135,13 +209,9 @@ export const COMPARISON_METRIC_KEYS                    = [
   'executorCpuTime', 'allocatedCoreHours',
 ];
 
-// Volume/count metrics, not cost metrics: more or less input/output data, or
-// tasks/executors, isn't inherently better or worse (it may just reflect a
-// differently-sized job), unlike wall-clock, spill, GC, etc. Exported as the
-// single source of truth for "which metrics have no better/worse direction",
-// shared by the view (RunComparison.tsx, for Δ coloring) and by
-// src/cli/budgets.ts's checkRegression (so a `--regression-metric inputBytes`
-// budget can't misread "processed more data" as a regression).
+// Volume/count metrics have no better/worse direction: more input, output, tasks or executors may
+// just reflect a differently-sized job. Shared by the view (Δ coloring) and by the CLI's
+// checkRegression, so a `--regression-metric inputBytes` budget cannot read more data as a regression.
 export const NEUTRAL_METRIC_KEYS                      = new Set(['inputBytes', 'outputBytes', 'taskCount', 'executorsAdded']);
 
 function direction(key        , baseline               , candidate               )                              {
@@ -151,16 +221,21 @@ function direction(key        , baseline               , candidate
   return d < 0 ? 'improvement' : d > 0 ? 'regression' : 'unchanged';
 }
 
-function metric(
-  key        , label        , baseline               , candidate               ,
-  extra                                 = {},
-)                 {
-  const dir = direction(key, baseline, candidate);
-  const delta = baseline == null || candidate == null ? null : candidate - baseline;
-  return { key, label, baseline, candidate, delta, direction: dir, ...extra };
-}
+/** A change under this share of the baseline's value is within run-to-run noise: a shared cluster
+ * easily moves a job a percent or two. The caller's `noiseBand` replaces it. */
+export const DEFAULT_NOISE_BAND = 0.02;
 
-export function metricDeltas(baseSnap                 , candSnap                 )                   {
+export function metricDeltas(baseSnap                 , candSnap                 , noiseBand = DEFAULT_NOISE_BAND)                   {
+  const metric = (
+    key        , label        , baseline               , candidate               ,
+    extra                                 = {},
+  )                 => {
+    const delta = baseline == null || candidate == null ? null : candidate - baseline;
+    return {
+      key, label, baseline, candidate, delta, direction: direction(key, baseline, candidate),
+      withinNoise: baseline != null && candidate != null && withinBand(baseline, candidate, noiseBand), ...extra,
+    };
+  };
   const out                   = [];
   // Run totals count every task attempt, failed and speculative ones too (run-totals.ts), the
   // same sums the CLI metrics block reports; skew stays on each stage's latest attempt.
@@ -238,44 +313,84 @@ export function metricDeltas(baseSnap                 , candSnap
   return out;
 }
 
-// Finding categories are counted, not matched: a (rule × impact band) tally needs
-// no cross-run stage identity, so it stays correct even when almost no stages
-// match uniquely (the common case on real logs, see plan rationale). A
-// category the candidate has more of is "introduced"; fewer, "resolved".
-export function findingsDelta(baseSnap                 , candSnap                 )   
-                                                               
-  {
-                                                                                                              
-  const tally = (snap                 )                          => {
-    const m = new Map                    (); // `${rule}§${impactBand}` -> { rule, impactBand, count, stages:Set }
-    for (const f of snap.catalog) {
-      const rule = 'rule' in f && typeof f.rule === 'string' ? f.rule : f.type;
-      const impactBand = f.impactBand ?? 'unknown';
-      const key = `${rule}§${impactBand}`;
-      const e = m.get(key) ?? { rule, type: f.type, impactBand, count: 0, stages: new Set        () };
-      e.count++;
-      // Resolve stageId → name on this snapshot only: a single-side lookup, so
-      // it needs no cross-run identity. App-level findings (stageId null) add none.
-      const stage = f.stageId != null ? snap.stages.get(f.stageId) : null;
-      if (stage?.name) e.stages.add(stage.name);
-      m.set(key, e);
+// Cross-run identity of a snapshot's stages and SQL executions: a stage (or execution) the aligner
+// paired takes one label on both sides; an unpaired one takes a side-neutral label, so the same problem on an unpaired stage in both runs still matches.
+function findingLocators(baseSnap                 , candSnap                 , stagePairs             )                                                 {
+  const baseStage = new Map                (), candStage = new Map                ();
+  const executionVotes = new Map                             (); // base execution -> cand execution -> paired stages
+  for (const pair of stagePairs) {
+    baseStage.set(pair.baseStageIds[0], pair.pairId);
+    candStage.set(pair.candStageIds[0], pair.pairId);
+    const baseExec = baseSnap.stages.get(pair.baseStageIds[0])?.sqlExecutionId;
+    const candExec = candSnap.stages.get(pair.candStageIds[0])?.sqlExecutionId;
+    if (baseExec == null || candExec == null) continue;
+    const votes = executionVotes.get(baseExec) ?? new Map                ();
+    votes.set(candExec, (votes.get(candExec) ?? 0) + 1);
+    executionVotes.set(baseExec, votes);
+  }
+  // An execution pairs with the candidate execution that holds most of its paired stages.
+  const executionPair = new Map                ();
+  for (const [baseExec, votes] of executionVotes) {
+    const [candExec] = [...votes].sort((a, b) => b[1] - a[1] || a[0] - b[0])[0];
+    executionPair.set(baseExec, candExec);
+  }
+  const pairedCandExecs = new Set(executionPair.values());
+  return {
+    base: { stage: (id) => baseStage.get(id) ?? 'unpaired', execution: (id) => executionPair.has(id) ? `c${executionPair.get(id)}` : 'unpaired' },
+    cand: { stage: (id) => candStage.get(id) ?? 'unpaired', execution: (id) => pairedCandExecs.has(id) ? `c${id}` : 'unpaired' },
+  };
+}
+
+// A finding is the same finding in both runs when `findingKey` says so (type, rule and location,
+// never value or band). One with a counterpart is not introduced or resolved: a different band
+// makes it `changed`. Each row aggregates findings of one rule and band, with the stage names
+// of the side the findings are on.
+export function findingsDelta(baseSnap                 , candSnap                 , stagePairs              = [])                {
+  const locators = findingLocators(baseSnap, candSnap, stagePairs);
+  const diff = diffFindings(baseSnap.catalog, candSnap.catalog, locators.base, locators.cand);
+  const stageNames = (snap                 , findings           )           => {
+    const names = new Set        ();
+    for (const f of findings) {
+      const name = f.stageId != null ? snap.stages.get(f.stageId)?.name : undefined;
+      if (name) names.add(name);
     }
-    return m;
+    return [...names].sort();
   };
-  const b = tally(baseSnap), c = tally(candSnap);
-  const introduced                     = [], resolved                     = [];
-  for (const key of [...new Set([...b.keys(), ...c.keys()])].sort()) {
-    const be = b.get(key), ce = c.get(key);
-    const baseCount = be?.count ?? 0;
-    const candCount = ce?.count ?? 0;
-    if (candCount === baseCount) continue;
-    const meta = ce ?? be ;
-    const more = candCount > baseCount ? ce  : be ; // the side with more supplies the labels
-    const row                   = { rule: meta.rule, type: meta.type, impactBand: meta.impactBand, baseCount, candCount,
-      delta: candCount - baseCount, stages: [...more.stages].sort() };
-    (candCount > baseCount ? introduced : resolved).push(row);
-  }
-  return { introduced, resolved };
+  const stageLabelsOf = (names          )           => [...new Set(names.map(stageDisplayLabel))].sort();
+  const baseKeyOf = findingKeyer(baseSnap), candKeyOf = findingKeyer(candSnap);
+  const keysOf = (findings           , keyOf                        )           => [...new Set(findings.map(keyOf))].sort();
+  const group =    (items     , keyOf                     )                       => {
+    const groups = new Map             ();
+    for (const item of items) {
+      const key = keyOf(item);
+      const members = groups.get(key);
+      if (members) members.push(item); else groups.set(key, [item]);
+    }
+    return [...groups].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  };
+  const rows = (findings           , snap                 , side                 )                     =>
+    group(findings, (f) => `${findingRule(f)}§${findingBand(f)}`).map(([, fs]) => {
+      const count = fs.length;
+      const stages = stageNames(snap, fs);
+      return {
+        rule: findingRule(fs[0]), type: fs[0].type, impactBand: findingBand(fs[0]),
+        baseCount: side === 'base' ? count : 0, candCount: side === 'cand' ? count : 0,
+        delta: side === 'cand' ? count : -count, stages, stageLabels: stageLabelsOf(stages),
+        keys: keysOf(fs, side === 'cand' ? candKeyOf : baseKeyOf),
+      };
+    });
+  const changed                       = group(
+    diff.changed, (c) => `${findingRule(c.candidate)}§${findingBand(c.baseline)}§${findingBand(c.candidate)}`,
+  ).map(([, cs]) => {
+    const stages = stageNames(candSnap, cs.map((c) => c.candidate));
+    return {
+      rule: findingRule(cs[0].candidate), type: cs[0].candidate.type,
+      baseImpactBand: findingBand(cs[0].baseline), candImpactBand: findingBand(cs[0].candidate),
+      count: cs.length, stages, stageLabels: stageLabelsOf(stages),
+      keys: keysOf(cs.map((c) => c.candidate), candKeyOf), baseKeys: keysOf(cs.map((c) => c.baseline), baseKeyOf),
+    };
+  });
+  return { introduced: rows(diff.introduced, candSnap, 'cand'), resolved: rows(diff.resolved, baseSnap, 'base'), changed };
 }
 
 function namesConflict(a                     , b                     )          {
@@ -340,16 +455,23 @@ const EMPTY_TASK_DATA = new Map                 ();
                                  
                                                                                                   
                                                                                                
-                                                                                                   
+                                                                                                 
                                                                                              
                                     
+                                                                                                 
+                                                                                                  
+                                                                
+                                 
+                                                                                                
+                                                                                           
+                           
+                                                                                      
+                                                                  
+                     
  
 
-// Wraps the analyze()-to-compareRuns() snapshot-building sequence shared by
-// the CLI's --baseline path and mcp-tools.ts's compareRuns tool: both need
-// captureSnapshot (with an empty taskDataCache, the interactive drill-down
-// cache, never read by compareRuns/matchStages/metricDeltas/findingsDelta)
-// for each side before diffing them.
+// Snapshot-building shared by the CLI's --baseline path and mcp-tools.ts's compareRuns tool. The
+// empty taskDataCache is the interactive drill-down cache, which comparison never reads.
 export function buildComparison(
   baseline                                                           ,
   candidate                                                           ,
@@ -373,7 +495,11 @@ export function compareRuns(
   options                 = {},
 )                    {
   const baseSnap = baseline.snapshot, candSnap = candidate.snapshot;
-  // The old exact matcher stays the source of `matchedCoverage`; pairs and the gate come from the aligner.
+  const noiseBand = options.noiseBand ?? DEFAULT_NOISE_BAND;
+  if (!Number.isFinite(noiseBand) || noiseBand < 0 || noiseBand > 1) {
+    throw new Error(`noiseBand must be a number from 0 to 1, got ${noiseBand}.`);
+  }
+  // `matchedCoverage` comes from the exact matcher; pairs and the gate come from the aligner.
   const match = matchStages(baseSnap, candSnap);
   const alignment = alignStages(baseSnap, candSnap, { normalizePath: compileNormalizePatterns(options.normalizePath) });
   const runtimeCoverage = alignment.runtimeCoverage;
@@ -398,24 +524,29 @@ export function compareRuns(
     : lowCoverage
     ? `Only ${share} of executor run time is in matched stages, so per-stage rows mostly compare unrelated work.`
     : null;
+  const metrics = metricDeltas(baseSnap, candSnap, noiseBand);
   return {
     baselineLabel: baseline.label, candidateLabel: candidate.label,
+    baselineApp: runAppIdentity(baseSnap.app), candidateApp: runAppIdentity(candSnap.app),
     confidence,
     reason,
     matchedCoverage: match.coverage,
     comparisonSchemaVersion: COMPARISON_SCHEMA_VERSION,
     runtimeCoverage,
-    stagePairs: alignment.pairs,
+    stagePairs: alignment.pairs.map((p) => ({
+      ...p, stageLabel: stageDisplayLabel(baseSnap.stages.get(p.baseStageIds[0])?.name ?? ''),
+    })),
     unmatched: alignment.unmatched,
     replanned: alignment.replanned,
     bookkeepingStageIds: alignment.bookkeepingStageIds,
     executionAlignment: alignment.executionAlignment,
-    metrics: metricDeltas(baseSnap, candSnap),
-    findings: findingsDelta(baseSnap, candSnap),
+    metrics,
+    findings: findingsDelta(baseSnap, candSnap, alignment.pairs),
     stageSkew: stageSkewDeltas(baseSnap, candSnap, alignment.pairs),
     baseStages: stageList(baseSnap),
     candStages: stageList(candSnap),
     jobOutcomes: { baseline: jobOutcome(baseSnap), candidate: jobOutcome(candSnap) },
+    comparability: buildComparability(baseSnap, candSnap, alignment, metrics, { ignoreConf: options.ignoreConf, confRedactRegex: options.confRedactRegex, noiseBand }),
   };
 }
 
@@ -429,6 +560,31 @@ function renderFindingsSection(title        , findings                    )
   return lines;
 }
 
+// Findings both runs have whose impact band moved: `### Changed findings (<count>)`, one bullet each.
+function renderChangedSection(changed                      )           {
+  const lines = [`### Changed findings (${changed.length})`, ''];
+  for (const f of changed) {
+    lines.push(`- ${f.rule}: ${f.baseImpactBand} -> ${f.candImpactBand}${f.count > 1 ? ` (${f.count} findings)` : ''}`);
+  }
+  return lines;
+}
+
+// What the runs share, as `- key: value` bullets: configuration, plan, data volume and executors.
+function renderComparabilitySection({ conf, plan, volume, capacity }               )           {
+  const confLine = conf.equal == null ? 'a run records no Spark properties'
+    : conf.equal ? 'same' : `${conf.differences.length} differ (${conf.differences.map((d) => d.key).join(', ')})`;
+  const pair = (f                                                       ) => `${f.baseline ?? 'n/a'} -> ${f.candidate ?? 'n/a'}`;
+  const execs = (side                                                                ) =>
+    `${side.meanExecutors == null ? 'n/a' : side.meanExecutors.toFixed(1)} mean / ${side.peakExecutors ?? 'n/a'} peak`;
+  return [
+    '### Comparability', '',
+    `- configuration: ${confLine}`,
+    `- plan: ${plan.equal ? 'every stage pairs' : `${plan.unmatched.baseline} baseline / ${plan.unmatched.candidate} candidate stages unmatched, ${plan.replanned} re-planned`}`,
+    `- input bytes: ${pair(volume.inputBytes)}, output rows: ${pair(volume.outputRows)}, output bytes: ${pair(volume.outputBytes)}`,
+    `- executors: ${execs(capacity.baseline)} -> ${execs(capacity.candidate)}`,
+  ];
+}
+
 // Small Markdown renderer for the comparison section, matching
 // evidence-report.ts's renderMarkdown house style (## section heading, ###
 // subheadings, `- key: value` bullets). Shared by the CLI's --baseline
@@ -458,6 +614,8 @@ export function renderComparisonMarkdown(
   lines.push(`- runtime coverage: ${comparison.runtimeCoverage === null ? 'n/a' : `${(comparison.runtimeCoverage * 100).toFixed(1)}%`}`);
   lines.push(`- stage pairs: ${comparison.stagePairs.length}, unmatched: ${comparison.unmatched.baseStageIds.length} baseline / ${comparison.unmatched.candStageIds.length} candidate, Delta bookkeeping: ${comparison.bookkeepingStageIds.baseStageIds.length} baseline / ${comparison.bookkeepingStageIds.candStageIds.length} candidate`);
   lines.push('');
+  lines.push(...renderComparabilitySection(comparison.comparability));
+  lines.push('');
   lines.push('### Metric deltas');
   lines.push('');
   for (const m of comparison.metrics) {
@@ -467,5 +625,7 @@ export function renderComparisonMarkdown(
   lines.push(...renderFindingsSection('Introduced findings', comparison.findings.introduced));
   lines.push('');
   lines.push(...renderFindingsSection('Resolved findings', comparison.findings.resolved));
+  lines.push('');
+  lines.push(...renderChangedSection(comparison.findings.changed));
   return lines.join('\n');
 }

package/vendor-core/run-identity.js

@@ -0,0 +1,12 @@
+                                               
+
+/** Which Spark application a report describes: its id and attempt id, null where the log has none
+ * (an attempt id is written only by a cluster-mode run). */
+                                 
+                    
+                           
+ 
+
+export function runAppIdentity(app                                 )                 {
+  return { id: app?.id ?? null, attemptId: typeof app?.attemptId === 'string' ? app.attemptId : null };
+}

package/vendor-core/run-interpretation.js

@@ -22,6 +22,7 @@ import {
 } from './run-verdict.js';
 import { recommendationText } from './finding-names.js';
 import { getScorecardEstimates } from './scorecard-estimates.js';
+import { singleStageId } from './finding-filter-predicate.js';
 import { checkConcurrentJobGroups } from './job-groups.js';
 import { computeWallClock } from './wall-clock.js';
 import { computeWastedCoreHours,                            } from './wasted-core-hours.js';
@@ -57,6 +58,15 @@ import { computeWastedCoreHours,                            } from './wasted-cor
                    
  
 
+/** One cost-lane entry: a finding ranked by the measured busy core time its fix removes.
+ * `leadIndex` names the finding. */
+                                      
+                    
+                         
+                                                 
+                 
+ 
+
                                  
                 
                     
@@ -64,7 +74,12 @@ import { computeWastedCoreHours,                            } from './wasted-cor
                  
                                                           
                                
+                                              
                            
+                                                                                             
+                                   
+                                                                                                    
+                          
                                                            
                     
                                                                
@@ -265,6 +280,12 @@ export function interpretRun(appModel          , catalog           , configFindi
       clean: verdict.facts.clean,
       failureReason: outcome.reason,
       steps,
+      costSteps: verdict.costLane.map(({ finding, figure }) => ({
+        leadIndex: indexOf.get(finding) ,
+        stageId: singleStageId(finding),
+        figure,
+      })),
+      idleLine: verdict.idle?.text ?? null,
       remaining: verdict.remaining,
       copyText: verdict.copyText,
     },

package/vendor-core/run-metrics.js

@@ -3,7 +3,7 @@
 // moves independently of the evidence report's. A figure the log cannot provide is null, never 0.
 import { computeAllocation,                 } from './allocation.js';
 import { computeSkewRatio, ENTRY_BY_TYPE,                         } from './detectors.js';
-import { totalExecutorCpuMs, withEarlierAttempts } from './run-totals.js';
+import { totalExecutorCpuMs, totalOutputRows, withEarlierAttempts } from './run-totals.js';
 import { isPythonStage } from './python-stage.js';
 import { stageIdentity } from './run-comparison.js';
 import { hasCompleteApplicationInterval } from './scorecard-estimates.js';
@@ -128,7 +128,7 @@ function stageMetrics(stages         , minTasksForP95        )               {
     shuffleWriteBytes: fromTasks((s) => s.shuffleWriteBytes),
     inputBytes: fromTasks((s) => s.inputBytes),
     outputBytes: fromTasks((s) => s.outputBytes),
-    outputRows: sumOf(finished, (s) => s.outputRecords),
+    outputRows: totalOutputRows(attempts),
     peakExecutionMemoryBytes: peakExecutionMemory(finished),
     taskCount: tasks != null && tasks > 0 ? tasks : null,
     failedTasks: fromTasks((s) => s.failedTasks),

package/vendor-core/run-output.js

@@ -1,26 +1,59 @@
-import { runFindings } from './evidence-report.js';
+import { cacheFor, runFindings,                  } from './evidence-report.js';
 import { redactRunModel } from './redact.js';
 import { computeRunMetrics,                 } from './run-metrics.js';
 import { buildEffectiveConf,                    } from './effective-conf.js';
+import { runAppIdentity,                     } from './run-identity.js';
                                                          
                                            
 
+/** Which build wrote a report: the host package's name and version, and the build id of the core it
+ * loaded. Core does not know its host, so the surface passes it in. */
+                                  
+               
+                  
+                  
+ 
+
+                                                                     
+
+// One entry per run model and threshold overrides, kept apart for the plain and the redacted model.
+// A WeakMap needs no invalidation: the entry is collectible once the model is (see runFindings).
+const plainBlocksCache                             = new WeakMap();
+const redactedBlocksCache                             = new WeakMap();
+
+// The app the blocks read (redacted under `redact`) and the run's `metrics`, the costly part,
+// computed once per run and options.
+function runBlocksBase(appModel          , redact                     , thresholds                                )                {
+  const cache = cacheFor(redact ? redactedBlocksCache : plainBlocksCache, thresholds);
+  const cached = cache.get(appModel);
+  if (cached) return cached;
+  let blocksModel = appModel;
+  if (redact) {
+    const { catalog, config } = runFindings(appModel, thresholds);
+    blocksModel = redactRunModel(appModel, catalog, config).appModel;
+  }
+  const base = { app: blocksModel.app, metrics: computeRunMetrics(blocksModel, thresholds) };
+  cache.set(appModel, base);
+  return base;
+}
+
 /** The CLI report's `metrics` and `effectiveConf` blocks, shared with MCP diagnose_run. Under
  * `redact` they come from the run redacted with the report's own inputs, so host and app
- * pseudonyms match the report's. `confKeys`/`confRedactRegex` narrow and mask `effectiveConf`. */
+ * pseudonyms match the report's. `confKeys`/`confRedactRegex` narrow and mask `effectiveConf`.
+ * `generator`, when the surface supplies it, is returned as given: it holds no run identifiers.
+ * `app` names the Spark application (redacted under `redact`), so two reports can be told apart. */
 export function runOutputBlocks(
   appModel          ,
-  { redact, thresholds, confKeys, confRedactRegex }   
+  { redact, thresholds, confKeys, confRedactRegex, generator }   
                                                                                                      
+                                
     = {},
-)                                                               {
-  let blocksModel = appModel;
-  if (redact) {
-    const { catalog, config } = runFindings(appModel, thresholds);
-    blocksModel = redactRunModel(appModel, catalog, config).appModel;
-  }
+)                                                                                                                 {
+  const { app, metrics } = runBlocksBase(appModel, redact, thresholds);
   return {
-    metrics: computeRunMetrics(blocksModel, thresholds),
-    effectiveConf: buildEffectiveConf(blocksModel.app, { keys: confKeys, userPattern: confRedactRegex }),
+    ...(generator ? { generator } : {}),
+    app: runAppIdentity(app),
+    metrics,
+    effectiveConf: buildEffectiveConf(app, { keys: confKeys, userPattern: confRedactRegex }),
   };
 }

package/vendor-core/run-totals.js

@@ -22,3 +22,15 @@ export function totalExecutorCpuMs(stages
   for (const s of stages) if (typeof s.executorCpuTime === 'number' && Number.isFinite(s.executorCpuTime) && s.executorCpuTime > 0) ns += s.executorCpuTime;
   return ns > 0 ? nsToMs(ns) : null;
 }
+
+/** Summed output rows over the given stage attempts (see `withEarlierAttempts`). Like the other
+ * task-derived totals it is null until some attempt finished a task, and null when no stage
+ * recorded a row count, never 0. */
+export function totalOutputRows(attempts                                              )                {
+  if (!attempts.some((s) => (s.taskCount ?? 0) > 0)) return null;
+  let sum = 0, present = false;
+  for (const s of attempts) {
+    if (typeof s.outputRecords === 'number' && Number.isFinite(s.outputRecords)) { sum += s.outputRecords; present = true; }
+  }
+  return present ? sum : null;
+}