sparkforensics-server 0.4.0 → 0.5.1

package/vendor-core/remediation.js

@@ -0,0 +1,20 @@
+// Constructors for the structured `remediation` a detector attaches next to its prose
+// `recommendation`: only where the detector already names the Spark property, never an invented one.
+                                                      
+
+                                          
+
+/** The property should be raised; `suggested` is the value when the detector computed one. */
+export function increaseConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'increase', suggested };
+}
+
+/** The property should be lowered; `suggested` is the value when the detector computed one. */
+export function decreaseConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'decrease', suggested };
+}
+
+/** The property should take a specific value (a switch or a class name), not move up or down. */
+export function setConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'set', suggested };
+}

package/vendor-core/run-comparison.js

@@ -1,9 +1,12 @@
 import { computeWallClock } from './wall-clock.js';
 import { normalizeDetail } from './detectors.js';
-import { cyrb53 } from './string-hash.js';
+import { normalizeStageName, stageIdentityWith, identityIndexWith, pairEqualCounts } from './stage-identity.js';
+import { computeAllocation } from './allocation.js';
+import { totalExecutorCpuMs, withEarlierAttempts } from './run-totals.js';
 import { captureSnapshot } from './session-snapshot.js';
+import { alignStages, compileNormalizePatterns, COMPARISON_SCHEMA_VERSION,                                     } from './stage-alignment.js';
 import { tunedRunNote } from './threshold-overrides.js';
-                                                                                                    
+                                                                                          
                                                              
                                                                                         
 import { isIncompleteRun } from './check-coverage.js';
@@ -18,13 +21,26 @@ import { summarizeRunOutcome } from './run-outcome.js';
                                    
                                                                                                                         
  
+                               
+                                                               
+                                                                          
+ 
                                     
                                                 
-                                                  
+                                                                                                
+                                                                   
+                                                                                                 
+                                                                         
                           
+                                                            
+                                 
+                          
+                                         
+                                         
+                                                             
                             
                                                                              
-                                                                                                                                                  
+                            
                                                                             
                                                                             
                                                                                             
@@ -38,73 +54,10 @@ function jobOutcome(snapshot                 )                    {
   return { failedJobs, totalJobs, incomplete: isIncompleteRun(snapshot.catalog) };
 }
 
-// Replace run-varying tokens (digit runs, long hex ids) with a stable marker so
-// the same logical stage across two runs normalizes to one identity.
-export function normalizeStageName(name        )         {
-  return String(name)
-    .toLowerCase()
-    .replace(/\b[0-9a-f]{8,}\b/g, '#') // hex ids/uuids first (they contain digits)
-    .replace(/\d+/g, '#')
-    .replace(/\s+/g, ' ')
-    .trim();
-}
-
-// Bottom-up, order-independent structural identity of a resolved plan tree:
-// each node folds its normalized name/detail with its children's digests
-// (children sorted, so AQE picking a different broadcast side still matches),
-// so two plans collide only when their whole shape and every node's detail
-// agree. `normalizeDetail` strips the run-to-run noise (expr ids, `plan_id=`,
-// codegen numbers, AQE build-side choice, commutative-operand order). Each
-// node folds to a fixed-length cyrb53 digest (JSON.stringify-encoded, so
-// detail text containing `<`/`>`/`{`/`,` can't collide two different plans)
-// instead of embedding full child identity strings, which would re-escape
-// every level below and blow identity size up to ~2^depth on the 20-60+
-// operator-deep plans real Spark produces.
-function planTreeIdentity(root                             )                {
-  if (!root) return null;
-  function visit(node          )         {
-    const childDigests = (node.children ?? []).map(visit).sort();
-    return cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? ''), childDigests]));
-  }
-  return visit(root);
-}
-
-// Plan identity for the stage's SQL execution, scoped to only the plan nodes
-// this stage actually ran (`node.stageIds`), not the whole tree: two stages
-// sharing one SQL execution (e.g. a self-join's two Exchange stages) otherwise
-// collapse onto one identity regardless of which part of the plan each ran.
-// Falls back to the coarser whole-tree identity when the stage has no
-// attributed nodes (hand-built snapshots without `stageIds`, or unmatched
-// accumulables).
-function sqlNodeIdentity(stage       , snapshot                 )         {
-  const execId = stage.sqlExecutionId;
-  if (execId == null) return '';
-  const root = snapshot.sql.get(execId)?.planTree ?? null;
-  if (!root) return '';
-  const fingerprints           = [];
-  (function collect(node          )       {
-    if (node.stageIds?.includes(stage.id)) {
-      fingerprints.push(JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
-    }
-    for (const child of node.children ?? []) collect(child);
-  })(root);
-  if (fingerprints.length === 0) return planTreeIdentity(root) ?? '';
-  return cyrb53(JSON.stringify(fingerprints.sort()));
-}
-
-export function stageIdentity(stage       , snapshot                 )         {
-  return normalizeStageName(stage.name ?? '') + '§' + sqlNodeIdentity(stage, snapshot);
-}
+export { normalizeStageName };
 
-function identityIndex(snapshot                 )                        {
-  const byIdentity = new Map                  ();
-  for (const [id, stage] of snapshot.stages) {
-    const key = stageIdentity(stage, snapshot);
-    const ids = byIdentity.get(key);
-    if (ids) ids.push(id);
-    else byIdentity.set(key, [id]);
-  }
-  return byIdentity;
+export function stageIdentity(stage       , snapshot                              )         {
+  return stageIdentityWith(stage, snapshot, normalizeDetail);
 }
 
 export function matchStages(baseSnap                 , candSnap                 )   
@@ -113,10 +66,8 @@ export function matchStages(baseSnap                 , candSnap
                                    
                    
   {
-  const baseIdx = identityIndex(baseSnap);
-  const candIdx = identityIndex(candSnap);
-  const pairs                                                              = [];
-  const matchedIdentities = new Set        ();
+  const baseIdx = identityIndexWith(baseSnap, (stage) => stageIdentity(stage, baseSnap));
+  const candIdx = identityIndexWith(candSnap, (stage) => stageIdentity(stage, candSnap));
   const collisionIdentities = new Set        ();
   // A collision is a single-run property: more than one stage in the SAME run
   // shares an identity. Recorded per-run regardless of whether the other run
@@ -124,26 +75,7 @@ export function matchStages(baseSnap                 , candSnap
   // don't, so this set means "was ambiguous", not "stayed unpaired".
   for (const idx of [baseIdx, candIdx])
     for (const [identity, ids] of idx) if (ids.length > 1) collisionIdentities.add(identity);
-  // An identity colliding equally on both sides has no genuine ambiguity about
-  // *count*, so pair its stages off positionally by sorted id rather than
-  // dropping them. This is exact when the identity actually distinguishes
-  // stages (e.g. self-comparing a run: every stage matches itself). It's a
-  // best-effort guess when the identity is coarse (no SQL/attribution) and
-  // the two sides are genuinely different runs -- two unrelated same-named
-  // stages could get cross-paired. Accepted tradeoff: dropping them instead
-  // would also sacrifice the exact-self-comparison case, which matters more.
-  for (const identity of [...baseIdx.keys()].sort()) {
-    const candIds = candIdx.get(identity);
-    if (!candIds) continue;
-    const baseIds = baseIdx.get(identity) ;
-    if (baseIds.length !== candIds.length) continue;
-    const sortedBaseIds = [...baseIds].sort((a, b) => a - b);
-    const sortedCandIds = [...candIds].sort((a, b) => a - b);
-    for (let i = 0; i < sortedBaseIds.length; i++) {
-      pairs.push({ identity, baseId: sortedBaseIds[i], candId: sortedCandIds[i] });
-    }
-    matchedIdentities.add(identity);
-  }
+  const { pairs, matchedIdentities } = pairEqualCounts(baseIdx, candIdx);
   const total = baseSnap.stages.size + candSnap.stages.size;
   // Both runs stage-less: nothing to compare, not "nothing matched".
   const coverage = total === 0 ? 1 : (2 * pairs.length) / total;
@@ -199,6 +131,7 @@ function skewRatios(stages         )           {
 export const COMPARISON_METRIC_KEYS                    = [
   'wallClock', 'shuffleSpill', 'taskSkew', 'failedTaskRate', 'diskSpill', 'gcTime',
   'inputBytes', 'outputBytes', 'executorRunTime', 'taskCount', 'executorsAdded',
+  'executorCpuTime', 'allocatedCoreHours',
 ];
 
 // Volume/count metrics, not cost metrics: more or less input/output data, or
@@ -228,20 +161,24 @@ function metric(
 
 export function metricDeltas(baseSnap                 , candSnap                 )                   {
   const out                   = [];
+  // Run totals count every task attempt, failed and speculative ones too (run-totals.ts), the
+  // same sums the CLI metrics block reports; skew stays on each stage's latest attempt.
+  const attemptsOf = (snap                 )          => withEarlierAttempts([...snap.stages.values()]);
 
   // Wall-clock: always computable (computeWallClock tolerates a null app).
   out.push(metric('wallClock', 'Wall-clock duration',
     computeWallClock(baseSnap.app, baseSnap.stages).total,
     computeWallClock(candSnap.app, candSnap.stages).total));
 
-  // Shuffle spill over the whole run: a sum needs no stage matching, and
-  // matching is unreliable on real logs (see plan rationale), so scope it to
-  // all stages exactly like task-skew and failed-rate below.
-  const bSpill = sumField([...baseSnap.stages.values()], 'memoryBytesSpilled');
-  const cSpill = sumField([...candSnap.stages.values()], 'memoryBytesSpilled');
-  out.push(metric('shuffleSpill', 'Shuffle spill',
+  // Memory spill (Spark's memoryBytesSpilled) over the whole run: a sum needs no
+  // stage matching, and matching is unreliable on real logs, so scope it to all
+  // stages exactly like task-skew and failed-rate below. The key stays
+  // `shuffleSpill` so existing --regression-metric callers keep working.
+  const bSpill = sumField(attemptsOf(baseSnap), 'memoryBytesSpilled');
+  const cSpill = sumField(attemptsOf(candSnap), 'memoryBytesSpilled');
+  out.push(metric('shuffleSpill', 'Memory spill',
     bSpill.present ? bSpill.sum : null, cSpill.present ? cSpill.sum : null,
-    { unavailableReason: bSpill.present && cSpill.present ? undefined : 'No shuffle-spill data recorded for a run' }));
+    { unavailableReason: bSpill.present && cSpill.present ? undefined : 'No memory-spill data recorded for a run' }));
 
   // Task skew: p95 of per-stage ratio across the whole run.
   const bSkew = p95(skewRatios([...baseSnap.stages.values()]));
@@ -250,10 +187,10 @@ export function metricDeltas(baseSnap                 , candSnap
     { unavailableReason: bSkew != null && cSkew != null ? undefined : 'No stage had measurable duration for a run' }));
 
   // Failed-task rate: Σ failedTasks / Σ taskCount.
-  const bTasks = sumField([...baseSnap.stages.values()], 'taskCount');
-  const cTasks = sumField([...candSnap.stages.values()], 'taskCount');
-  const bFailed = sumField([...baseSnap.stages.values()], 'failedTasks');
-  const cFailed = sumField([...candSnap.stages.values()], 'failedTasks');
+  const bTasks = sumField(attemptsOf(baseSnap), 'taskCount');
+  const cTasks = sumField(attemptsOf(candSnap), 'taskCount');
+  const bFailed = sumField(attemptsOf(baseSnap), 'failedTasks');
+  const cFailed = sumField(attemptsOf(candSnap), 'failedTasks');
   // Guard on BOTH inputs: a missing `failedTasks` field must render Unavailable,
   // not a false 0% rate (dividing an absent-and-therefore-0 numerator).
   const bRate = bTasks.present && bTasks.sum > 0 && bFailed.present ? bFailed.sum / bTasks.sum : null;
@@ -265,8 +202,8 @@ export function metricDeltas(baseSnap                 , candSnap
   // carries (set in finalizeStage). Correct at any match coverage, like the
   // sums above; no parser or detector change.
   const sumMetric = (key        , label        , field                   , reason        ) => {
-    const b = sumField([...baseSnap.stages.values()], field);
-    const c = sumField([...candSnap.stages.values()], field);
+    const b = sumField(attemptsOf(baseSnap), field);
+    const c = sumField(attemptsOf(candSnap), field);
     out.push(metric(key, label, b.present ? b.sum : null, c.present ? c.sum : null,
       { unavailableReason: b.present && c.present ? undefined : reason }));
   };
@@ -285,6 +222,18 @@ export function metricDeltas(baseSnap                 , candSnap
   out.push(metric('executorsAdded', 'Executors added', bExec, cExec,
     { unavailableReason: bExec != null && cExec != null ? undefined : 'No executor events recorded for a run' }));
 
+  // Executor CPU time (ms) and allocated core-hours cost resources, so less is better. CPU time is
+  // null, not 0, on a run whose log never recorded it (older Spark).
+  const cpuMs = (snap                 )                => totalExecutorCpuMs(attemptsOf(snap));
+  const bCpu = cpuMs(baseSnap), cCpu = cpuMs(candSnap);
+  out.push(metric('executorCpuTime', 'Executor CPU time', bCpu, cCpu,
+    { unavailableReason: bCpu != null && cCpu != null ? undefined : 'No executor CPU time recorded for a run' }));
+  const coreHours = (snap                 )                =>
+    (snap.executors ? computeAllocation(snap).coreHours : null);
+  const bCore = coreHours(baseSnap), cCore = coreHours(candSnap);
+  out.push(metric('allocatedCoreHours', 'Allocated core-hours', bCore, cCore,
+    { unavailableReason: bCore != null && cCore != null ? undefined : 'No executor lifecycle or core count recorded for a run' }));
+
   return out;
 }
 
@@ -333,19 +282,19 @@ function namesConflict(a                     , b                     )
 }
 
 // A pair's skew ratio is null when the stage's duration wasn't measurable (see
-// `stageSkewRatio`). `baseId`/`candId` are carried through so a consumer can
-// key rows uniquely: two pairs can share one `identity` (a same-run collision
-// resolved positionally in matchStages), but never the same baseId+candId.
-function stageSkewDeltas(
-  baseSnap                 ,
-  candSnap                 ,
-  match                                                                        ,
-)                                                                                                                                       {
-  return match.pairs.map((p) => {
-    const b = stageSkewRatio(baseSnap.stages.get(p.baseId) );
-    const c = stageSkewRatio(candSnap.stages.get(p.candId) );
-    return { identity: p.identity, baseId: p.baseId, candId: p.candId, baseline: b, candidate: c, delta: b != null && c != null ? c - b : null };
-  }).sort((x, y) => x.identity < y.identity ? -1 : x.identity > y.identity ? 1 : 0);
+// `stageSkewRatio`). Rows come from the aligner's `stagePairs`, so the table and the coverage
+// banner agree. A pair covers one stage per side here; `pairId` is the unique row key.
+function stageSkewDeltas(baseSnap                 , candSnap                 , stagePairs             )                 {
+  return stagePairs.map((p) => {
+    const baseId = p.baseStageIds[0], candId = p.candStageIds[0];
+    const baseStage = baseSnap.stages.get(baseId) ;
+    const b = stageSkewRatio(baseStage);
+    const c = stageSkewRatio(candSnap.stages.get(candId) );
+    return {
+      pairId: p.pairId, name: normalizeStageName(baseStage.name ?? ''), baseId, candId,
+      baseline: b, candidate: c, delta: b != null && c != null ? c - b : null,
+    };
+  }).sort((x, y) => x.name < y.name ? -1 : x.name > y.name ? 1 : x.baseId - y.baseId);
 }
 
 // Compact, view-friendly per-stage record for the manual stage-pinning panel.
@@ -387,6 +336,14 @@ function stageList(snap                 )
 // caller's reference, so one shared empty Map is safe here.
 const EMPTY_TASK_DATA = new Map                 ();
 
+                                 
+                                                                                                  
+                                                                                               
+                                                                                                   
+                                                                                             
+                                    
+ 
+
 // Wraps the analyze()-to-compareRuns() snapshot-building sequence shared by
 // the CLI's --baseline path and mcp-tools.ts's compareRuns tool: both need
 // captureSnapshot (with an empty taskDataCache, the interactive drill-down
@@ -395,50 +352,65 @@ const EMPTY_TASK_DATA = new Map                 ();
 export function buildComparison(
   baseline                                                           ,
   candidate                                                           ,
+  options                 ,
 )                    {
   return compareRuns(
     { label: baseline.label, snapshot: captureSnapshot(baseline.appModel, baseline.catalog, EMPTY_TASK_DATA) },
     { label: candidate.label, snapshot: captureSnapshot(candidate.appModel, candidate.catalog, EMPTY_TASK_DATA) },
+    options,
   );
 }
 
-// matchStages' coverage is a Dice coefficient: (2 * pairs.length) / (baseCount
-// + candCount). Below 0.5, more than half of each run's stages went unpaired,
-// so the stage-level rows (stageSkew, baseStages/candStages) mostly show
-// unrelated work side by side rather than the same stage before/after -- the
-// comparison is dominated by guesswork, not genuine pairing. 0.5 is thus the
-// natural midpoint for "more matched than not," not an arbitrary tuning knob.
-const LOW_COVERAGE_THRESHOLD = 0.5;
+// `ok` needs this share of both runs' executor run time to sit in paired stages. Run time, not
+// stage count: the stages a run spends its time in decide whether a per-stage delta compares the
+// same work, and a count lets many small matched stages hide an unmatched heavy one.
+const RUNTIME_COVERAGE_THRESHOLD = 0.9;
 
 export function compareRuns(
   baseline                                              ,
   candidate                                              ,
+  options                 = {},
 )                    {
   const baseSnap = baseline.snapshot, candSnap = candidate.snapshot;
+  // The old exact matcher stays the source of `matchedCoverage`; pairs and the gate come from the aligner.
   const match = matchStages(baseSnap, candSnap);
+  const alignment = alignStages(baseSnap, candSnap, { normalizePath: compileNormalizePatterns(options.normalizePath) });
+  const runtimeCoverage = alignment.runtimeCoverage;
   // Name equality is a weak confidence signal, not a hard gate: renaming a job
   // is the normal way to label an A/B experiment, so a mismatch must not block
   // the (matching-free, name-independent) deltas. Surface it as `low` instead.
   const namesDiffer = namesConflict(baseSnap.app, candSnap.app);
-  // Coverage is the other half of the signal: identical names on two runs that
-  // barely share any stages are just as misleading as differing names on two
-  // runs that match well, so either condition alone drops confidence to `low`.
-  const lowCoverage = match.coverage < LOW_COVERAGE_THRESHOLD;
-  const reason = namesDiffer && lowCoverage
-    ? `Run names differ and only ${(match.coverage * 100).toFixed(0)}% of stages matched, so deltas may compare different work.`
+  // Runtime coverage is the other half of the signal: identical names on two runs whose heavy
+  // stages do not pair are just as misleading as differing names on two runs that pair well, so
+  // either condition alone drops confidence to `low`. No run time at all is `insufficient`.
+  const insufficient = runtimeCoverage === null;
+  const lowCoverage = runtimeCoverage !== null && runtimeCoverage < RUNTIME_COVERAGE_THRESHOLD;
+  const confidence                                  = insufficient ? 'insufficient' : namesDiffer || lowCoverage ? 'low' : 'ok';
+  // Rounded down, so a share just under the gate never reads as 90%.
+  const share = runtimeCoverage === null ? '' : `${Math.floor(runtimeCoverage * 100)}%`;
+  const reason = insufficient
+    ? `${namesDiffer ? 'Run names differ and no' : 'No'} executor run time was recorded in either run, so there is no work to compare.`
+    : namesDiffer && lowCoverage
+    ? `Run names differ and only ${share} of executor run time is in matched stages, so deltas may compare different work.`
     : namesDiffer
     ? 'Run names differ, so deltas may compare different work.'
     : lowCoverage
-    ? `Only ${(match.coverage * 100).toFixed(0)}% of stages matched between runs, so per-stage rows mostly compare unrelated work.`
+    ? `Only ${share} of executor run time is in matched stages, so per-stage rows mostly compare unrelated work.`
     : null;
   return {
     baselineLabel: baseline.label, candidateLabel: candidate.label,
-    confidence: namesDiffer || lowCoverage ? 'low' : 'ok',
+    confidence,
     reason,
     matchedCoverage: match.coverage,
+    comparisonSchemaVersion: COMPARISON_SCHEMA_VERSION,
+    runtimeCoverage,
+    stagePairs: alignment.pairs,
+    unmatched: alignment.unmatched,
+    replanned: alignment.replanned,
+    bookkeepingStageIds: alignment.bookkeepingStageIds,
     metrics: metricDeltas(baseSnap, candSnap),
     findings: findingsDelta(baseSnap, candSnap),
-    stageSkew: stageSkewDeltas(baseSnap, candSnap, match),
+    stageSkew: stageSkewDeltas(baseSnap, candSnap, alignment.pairs),
     baseStages: stageList(baseSnap),
     candStages: stageList(candSnap),
     jobOutcomes: { baseline: jobOutcome(baseSnap), candidate: jobOutcome(candSnap) },
@@ -467,16 +439,22 @@ export function renderComparisonMarkdown(
   const lines = ['', '## Comparison to baseline', ''];
   if (tuned) lines.push(`- Tuned thresholds (both runs): ${tunedRunNote(tuned)}`, '');
   if (verdict) {
-    // The dashboard comparison page's headline: run A is the baseline, run B the candidate.
-    lines.push(`Run A: ${comparison.baselineLabel} · Run B: ${comparison.candidateLabel}`, '', verdict.title);
+    // Names each run by its label (MCP's run IDs), unless the labels are just the role names the
+    // CLI passes, where the verdict below already says baseline and candidate.
+    if (comparison.baselineLabel !== 'baseline' || comparison.candidateLabel !== 'candidate') {
+      lines.push(`Baseline: ${comparison.baselineLabel} · Candidate: ${comparison.candidateLabel}`, '');
+    }
+    lines.push(verdict.title);
     if (verdict.sentences.length > 0) lines.push('', verdict.sentences.join(' '));
     lines.push('');
   }
-  if (comparison.confidence === 'low') {
-    lines.push(`- confidence: low, ${comparison.reason}`);
+  if (comparison.confidence !== 'ok') {
+    lines.push(`- confidence: ${comparison.confidence}, ${comparison.reason}`);
     lines.push('');
   }
   lines.push(`- matched stage coverage: ${(comparison.matchedCoverage * 100).toFixed(1)}%`);
+  lines.push(`- runtime coverage: ${comparison.runtimeCoverage === null ? 'n/a' : `${(comparison.runtimeCoverage * 100).toFixed(1)}%`}`);
+  lines.push(`- stage pairs: ${comparison.stagePairs.length}, unmatched: ${comparison.unmatched.baseStageIds.length} baseline / ${comparison.unmatched.candStageIds.length} candidate, Delta bookkeeping: ${comparison.bookkeepingStageIds.baseStageIds.length} baseline / ${comparison.bookkeepingStageIds.candStageIds.length} candidate`);
   lines.push('');
   lines.push('### Metric deltas');
   lines.push('');

package/vendor-core/run-interpretation.js

@@ -192,6 +192,7 @@ function interpretEfficiency(appModel          )                        {
     app: appModel.app,
     stages: appModel.stages,
     executorsAdded: appModel.executors.added,
+    executorsRemoved: appModel.executors.removed,
     runAggregates: appModel.runAggregates,
   });
 }
@@ -270,7 +271,7 @@ export function interpretRun(appModel          , catalog           , configFindi
     coverage: interpretCoverage(appModel, allFindings),
     runShape: interpretRunShape(appModel),
     wallClock: computeWallClock(appModel.app, appModel.stages),
-    wastedCoreHours: computeWastedCoreHours(appModel.app, appModel.executors.added, appModel.runAggregates),
+    wastedCoreHours: computeWastedCoreHours(appModel.app, appModel.executors.added, appModel.runAggregates, appModel.executors.removed),
     efficiency: interpretEfficiency(appModel),
     wallClockReliable: checkConcurrentJobGroups(appModel.jobs).wallClockReliable,
     etlPhases: attributeEtlPhases(appModel.stages),

package/vendor-core/run-metrics.js

@@ -0,0 +1,198 @@
+// The machine-readable metrics block of the CLI's JSON output: run-level totals and per-stage rows
+// an automated tuning loop can compare across runs without reading a report. Its schemaVersion
+// moves independently of the evidence report's. A figure the log cannot provide is null, never 0.
+import { computeAllocation,                 } from './allocation.js';
+import { computeSkewRatio, ENTRY_BY_TYPE,                         } from './detectors.js';
+import { totalExecutorCpuMs, withEarlierAttempts } from './run-totals.js';
+import { isPythonStage } from './python-stage.js';
+import { stageIdentity } from './run-comparison.js';
+import { hasCompleteApplicationInterval } from './scorecard-estimates.js';
+import { effectiveThresholds } from './threshold-overrides.js';
+import { computeWallClock } from './wall-clock.js';
+                                                  
+
+export const METRICS_SCHEMA_VERSION = 1;
+
+                               
+                            
+                                   
+                                   
+                          
+                                  
+                                
+                                  
+                                   
+                            
+                             
+                            
+                                          
+                           
+                             
+                              
+                      
+ 
+
+                                                       
+                                                                                                  
+                     
+                                                                                               
+                                 
+                         
+                          
+                  
+ 
+
+                             
+                        
+                                                                                                  
+                                                                                               
+                       
+                                                                                                                                    
+         
+                                                                   
+                                                                      
+                                                                                     
+                                            
+    
+          
+                                                 
+                                                                                                 
+                                                                                       
+                                                                     
+                                                                                    
+    
+                         
+           
+                                                                                             
+                                      
+    
+                                          
+ 
+
+const finite = (v         )              => typeof v === 'number' && Number.isFinite(v);
+
+// Σ of a field over the stages that carry a finite value; null when none does.
+function sumOf(stages         , pick                                         )                {
+  let sum = 0, present = false;
+  for (const s of stages) {
+    const v = pick(s);
+    if (finite(v)) { sum += v; present = true; }
+  }
+  return present ? sum : null;
+}
+
+// 0 means either no execution memory used or not recorded; only a positive peak is reported.
+function peakExecutionMemory(stages         )                {
+  const peaks = stages.map((s) => s.peakExecutionMemoryMax).filter((v)              => finite(v) && v > 0);
+  return peaks.length > 0 ? Math.max(...peaks) : null;
+}
+
+function maxSkew(stages         , minTasksForP95        )                {
+  const ratios = stages
+    .map((s) => computeSkewRatio(s, minTasksForP95)?.ratio)
+    .filter((r)              => finite(r));
+  return ratios.length > 0 ? Math.max(...ratios) : null;
+}
+
+// Failed stage attempts and stages submitted more than once, from the parser's per-stage attempt
+// counts; null when any stage record lacks them.
+function stageAttempts(stages         )                                             {
+  let failed = 0, retried = 0;
+  for (const s of stages) {
+    if (!finite(s.stageAttempts) || !finite(s.failedStageAttempts)) return null;
+    failed += s.failedStageAttempts;
+    if (s.stageAttempts > 1) retried++;
+  }
+  return { failed, retried };
+}
+
+// Metrics of a set of stages, every attempt included; a row folds the stages sharing one
+// fingerprint. Skew describes the latest attempt's task durations.
+function stageMetrics(stages         , minTasksForP95        )               {
+  const attempts = withEarlierAttempts(stages);
+  // Task-derived figures are null for stages that never finished (no task records).
+  // Superseded attempts carry work but no task count of their own, so they stay in once any
+  // attempt finished a task.
+  const finished = attempts.some((s) => (s.taskCount ?? 0) > 0) ? attempts : [];
+  const tasks = sumOf(attempts, (s) => s.taskCount);
+  const fromTasks = (pick                                  )                => (finished.length > 0 ? sumOf(finished, pick) : null);
+  const durations = attempts.filter((s) => finite(s.submittedAt) && finite(s.completedAt) && (s.completedAt          ) >= (s.submittedAt          ));
+  return {
+    durationMs: durations.length > 0 ? sumOf(durations, (s) => (s.completedAt          ) - (s.submittedAt          )) : null,
+    executorCpuTimeMs: totalExecutorCpuMs(finished),
+    executorRunTimeMs: fromTasks((s) => s.executorRunTime),
+    gcTimeMs: fromTasks((s) => s.jvmGCTime),
+    memorySpillBytes: fromTasks((s) => s.memoryBytesSpilled),
+    diskSpillBytes: fromTasks((s) => s.diskBytesSpilled),
+    shuffleReadBytes: fromTasks((s) => s.shuffleReadBytes),
+    shuffleWriteBytes: fromTasks((s) => s.shuffleWriteBytes),
+    inputBytes: fromTasks((s) => s.inputBytes),
+    outputBytes: fromTasks((s) => s.outputBytes),
+    outputRows: sumOf(finished, (s) => s.outputRecords),
+    peakExecutionMemoryBytes: peakExecutionMemory(finished),
+    taskCount: tasks != null && tasks > 0 ? tasks : null,
+    failedTasks: fromTasks((s) => s.failedTasks),
+    retriedTasks: fromTasks((s) => (s.wastedAttempts                      ) ?? 0),
+    skew: maxSkew(stages.filter((s) => (s.taskCount ?? 0) > 0), minTasksForP95),
+  };
+}
+
+/** The run's metrics block. `thresholds` only moves the skew ratio's P95 cutoff, as for the
+ * --max-skew budget. */
+export function computeRunMetrics(appModel          , thresholds                     )             {
+  const stageList = [...appModel.stages.values()];
+  const minTasksForP95 = effectiveThresholds(ENTRY_BY_TYPE.get('skew') , thresholds).minTasksForP95          ;
+  const all = stageMetrics(stageList, minTasksForP95);
+  const python = stageList.filter((s) => isPythonStage(s, appModel.sql));
+  const pythonRunTimeMs = sumOf(withEarlierAttempts(python), (s) => s.executorRunTime);
+
+  const byFingerprint = new Map                 ();
+  for (const s of stageList) {
+    const key = stageIdentity(s, appModel);
+    const group = byFingerprint.get(key);
+    if (group) group.push(s); else byFingerprint.set(key, [s]);
+  }
+  const stages                                  = {};
+  for (const [key, group] of byFingerprint) {
+    const attempts = stageAttempts(group);
+    stages[key] = {
+      stageIds: group.map((s) => s.id).sort((a, b) => a - b),
+      failed: attempts ? attempts.failed > 0 : null,
+      retried: attempts ? attempts.retried > 0 : null,
+      python: group.some((s) => isPythonStage(s, appModel.sql)),
+      ...stageMetrics(group, minTasksForP95),
+    };
+  }
+
+  const attempts = stageAttempts(stageList);
+  return {
+    schemaVersion: METRICS_SCHEMA_VERSION,
+    runComplete: appModel.app?.endTime != null,
+    time: {
+      wallClockMs: hasCompleteApplicationInterval(appModel.app) ? computeWallClock(appModel.app, appModel.stages).total : null,
+      executorCpuTimeMs: all.executorCpuTimeMs,
+      executorRunTimeMs: all.executorRunTimeMs,
+      gcTimeMs: all.gcTimeMs,
+    },
+    data: {
+      memorySpillBytes: all.memorySpillBytes, diskSpillBytes: all.diskSpillBytes,
+      shuffleReadBytes: all.shuffleReadBytes, shuffleWriteBytes: all.shuffleWriteBytes,
+      inputBytes: all.inputBytes, outputBytes: all.outputBytes, outputRows: all.outputRows,
+      peakExecutionMemoryBytes: all.peakExecutionMemoryBytes,
+    },
+    shape: {
+      taskCount: all.taskCount,
+      stageCount: stageList.length,
+      failedStageAttempts: attempts?.failed ?? null,
+      retriedStages: attempts?.retried ?? null,
+      failedTasks: all.failedTasks,
+      retriedTasks: all.retriedTasks,
+      maxSkew: all.skew,
+    },
+    allocation: computeAllocation(appModel),
+    python: {
+      shareOfTaskRunTime: all.executorRunTimeMs != null && all.executorRunTimeMs > 0 ? (pythonRunTimeMs ?? 0) / all.executorRunTimeMs : null,
+    },
+    stages,
+  };
+}