sparkforensics-server 0.5.2 → 0.8.1

package/vendor-core/run-verdict.js

@@ -6,17 +6,21 @@ import { hasFinishedStage, isCleanRun } from './check-coverage.js';
 import { findingActionLabel } from './finding-action-label.js';
 import { singleStageId } from './finding-filter-predicate.js';
 import { recommendationText } from './finding-names.js';
-import { formatDuration, IMPACT_BAND_ORDER } from './format-utils.js';
+import { formatDuration, formatRawWaste, IMPACT_BAND_ORDER, readsAsZero } from './format-utils.js';
 import { impactFigure, savingsMeaning } from './impact-format.js';
 import { isEligible } from './recommendation-rollup.js';
 import { FAILURE_TYPES, quotesReasonOf, summarizeRunOutcome,                 } from './run-outcome.js';
 import { getScorecardEstimates, hasCompleteApplicationInterval } from './scorecard-estimates.js';
 import { computeWallClock } from './wall-clock.js';
+import { computeWastedCoreHours } from './wasted-core-hours.js';
                                                     
 
 /** How many next steps the verdict lists before pointing at the full list. */
 export const NEXT_STEP_LIMIT = 3;
 
+/** How many cost-lane entries the verdict lists. */
+export const COST_LANE_LIMIT = 3;
+
 /** Finding types whose widget sits in the board's reference region (listed after every action
  * widget). The dashboard's registry must agree: tests/view/detector-registry.test.tsx checks it. */
 export const REFERENCE_DISPLAY_TYPES                      = new Set([
@@ -69,6 +73,56 @@ export function rankBySavings(findings           )            {
   return candidates.map(({ finding }) => finding);
 }
 
+/** One entry of the cost lane: a finding and the measured busy core time its fix removes. */
+                                
+                   
+                                                                
+                     
+                                                
+                 
+ 
+
+/** The cost lane: rankable findings ordered by measured busy core time (`coreTimeMs.high`), the
+ * one unit all of them share. An idle capacity figure, a modeled figure and a bytes-only figure
+ * never carry `coreTimeMs`, and a finding without one is in neither lane. Ties fall back to impact
+ * band, widget display order, then input order, like `rankBySavings`. */
+export function buildCostLane(findings           )                  {
+  const entries = findings
+    .map((finding, index) => ({ finding, index, coreTimeMs: finding.impactEstimate?.coreTimeMs?.high }))
+    .filter((entry)                                                                   =>
+      isRankable(entry.finding) && typeof entry.coreTimeMs === 'number' && Number.isFinite(entry.coreTimeMs) && entry.coreTimeMs > 0);
+  entries.sort((left, right) => (
+    right.coreTimeMs - left.coreTimeMs
+    || (IMPACT_BAND_ORDER[left.finding.impactBand] ?? 9) - (IMPACT_BAND_ORDER[right.finding.impactBand] ?? 9)
+    || (DISPLAY_INDEX.get(left.finding.type) ?? Number.MAX_SAFE_INTEGER) - (DISPLAY_INDEX.get(right.finding.type) ?? Number.MAX_SAFE_INTEGER)
+    || left.index - right.index
+  ));
+  return entries
+    .map(({ finding, coreTimeMs }) => ({ finding, coreTimeMs, figure: formatRawWaste({ value: coreTimeMs, unit: 'coreMs' }) }))
+    .filter((entry) => !readsAsZero(entry.figure));
+}
+
+/** The run's idle capacity, bounded by the run: allocated core time that ran no task. */
+                              
+                    
+                             
+                                                
+              
+                                                           
+               
+ 
+
+export function buildRunIdleLine(appModel          )                     {
+  const { wastedCoreHours, totalCoreHours } = computeWastedCoreHours(
+    appModel.app, appModel.executors.added, appModel.runAggregates, appModel.executors.removed,
+  );
+  if (wastedCoreHours == null || totalCoreHours == null || totalCoreHours <= 0) return null;
+  const figure = formatRawWaste({ value: wastedCoreHours, unit: 'coreHours' });
+  if (readsAsZero(figure)) return null;
+  const pct = Math.round((wastedCoreHours / totalCoreHours) * 100);
+  return { coreHours: wastedCoreHours, allocatedCoreHours: totalCoreHours, pct, text: `${figure} (${pct}%) of the allocation ran no task.` };
+}
+
 /** One place worth a look: the highest-ranked finding at a location, plus the
  * other finding types flagged at that same location. Findings that share a
  * stage usually share one root cause, so they read as one step, not several. */
@@ -179,6 +233,9 @@ function plural(count        , noun        )         {
                                                                         
                             
                             
+                                                                                                    
+                                                                           
+                     
  
 
 function isFailedRun(facts          )          {
@@ -215,6 +272,13 @@ export function verdictTitle(eligible           , steps            , facts
   return `Start here: ${lowerFirst(findingActionLabel(lead.lead))}`;
 }
 
+/** What the finding count leaves out, as a sentence tail: notes about an incomplete log or data the
+ * log lacked are not fixes, so the count does not include them. */
+function uncountedNote(uncounted                    )         {
+  if (!uncounted) return '';
+  return ` The count leaves out ${plural(uncounted, 'note')} on incomplete or missing log data.`;
+}
+
 /** The run-level summary under the title: how much was found and where, what
  * the first fix is worth, and the run's idle capacity when that is large
  * enough to matter but is not the first step (whose title already says it). */
@@ -234,7 +298,7 @@ export function verdictSummary(eligible           , steps            , facts
   } else if (steps.length === 0) {
     sentences.push('They are listed by impact under Findings.');
   } else {
-    sentences.push(`${plural(eligible.length, 'finding')} in ${plural(steps.length, 'place')}.`);
+    sentences.push(`${plural(eligible.length, 'finding')} in ${plural(steps.length, 'place')}.${uncountedNote(facts.uncounted)}`);
     const wallClock = steps[0].lead.impactEstimate?.wallClock;
     // An idle-capacity lead needs no sentence here: the title gives the idle share and step 1 the fix.
     if (!isIdleCapacityStep(steps[0]) && wallClock && facts.runMs != null) {
@@ -323,6 +387,10 @@ export function stepCopyRecommendation(step          , outcome            )
                     
                                                                                 
                           
+                                                                                                
+                            
+                                                                                
+                           
  
 
 export function buildRunVerdict(appModel          , allFindings           )                  {
@@ -336,6 +404,7 @@ export function buildRunVerdict(appModel          , allFindings           )
     clean: isCleanRun(appModel, allFindings),
     outcome,
     noFinishedStages: !hasFinishedStage(appModel.stages),
+    uncounted: allFindings.length - eligible.length,
   };
   const title = verdictTitle(eligible, steps, facts);
   const shown = steps.slice(0, NEXT_STEP_LIMIT);
@@ -348,5 +417,9 @@ export function buildRunVerdict(appModel          , allFindings           )
       remaining,
     })
     : null;
-  return { eligible, outcome, steps, facts, title, summary: verdictSummary(eligible, steps, facts), shown, remaining, copyText };
+  return {
+    eligible, outcome, steps, facts, title, summary: verdictSummary(eligible, steps, facts), shown, remaining, copyText,
+    costLane: buildCostLane(eligible).slice(0, COST_LANE_LIMIT),
+    idle: buildRunIdleLine(appModel),
+  };
 }

package/vendor-core/scaling-sim.js

@@ -1,7 +1,6 @@
-// §4 What-if executor-scaling simulator.
-// DESIGN SPIKE: the makespan model is a first-cut approximation and is NOT
-// validated against ground truth (a single event log only ever observes one
-// scale). Every prediction ships with a Model Error confidence indicator.
+// What-if executor-scaling simulator. The makespan model is a first-cut approximation and is NOT
+// validated against ground truth (a single event log only ever observes one scale). Every
+// prediction ships with a Model Error confidence indicator.
 //
 // Builds on computeCoreTimeSeries' clamp conceptually, but operates on the
 // worker-posted per-stage aggregates (runAggregates.perStage) so raw task data

package/vendor-core/snappy-block.js

@@ -5,7 +5,7 @@
 // Each raw block is the standard Snappy format (varint uncompressed length +
 // a sequence of literal/copy tagged elements): see
 // https://github.com/google/snappy/blob/main/format_description.txt. This is
-// a different framing from Spark's own LZ4Block format (src/lz4-block.js).
+// a different framing from Spark's own LZ4Block format (lz4-block.ts).
 
 const MAGIC = [0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0x00]; // 0x82 'S' 'N' 'A' 'P' 'P' 'Y' 0x00
 const HEADER_SIZE = 16; // 8-byte magic + 4-byte version + 4-byte compatible version
@@ -102,7 +102,7 @@ export function decodeSnappyBlock(bytes            )             {
 
 // Streaming counterpart: push arbitrary byte slices, each fully-received
 // block is decompressed and handed to `onChunk` as it completes. Mirrors
-// createLz4BlockDecoder's shape/contract (see src/lz4-block.js).
+// createLz4BlockDecoder's shape/contract (see lz4-block.ts).
 export function createSnappyBlockDecoder(
   onChunk                             ,
 )                                                 {

package/vendor-core/spark-conf.js

@@ -0,0 +1,168 @@
+// The Spark configuration a run actually executed under, resolved key by key from three layers:
+//   1. a per-query value: `SparkListenerSQLExecutionStart.modifiedConfigs`, the session settings of
+//      one SQL execution that differ from the SparkContext's (what `spark.conf.set` leaves),
+//   2. the app's Spark Properties from `SparkListenerEnvironmentUpdate`,
+//   3. Spark's own default for the run's Spark version (SPARK_DEFAULTS below).
+// A detector scoped to one SQL execution passes that execution's modifiedConfigs; any other caller
+// leaves it out and gets the run-wide answer.
+//
+// Properties with no entry in SPARK_DEFAULTS have no fixed default: spark.executor.instances and
+// spark.default.parallelism depend on the cluster manager and the cluster, spark.executor.memoryOverhead
+// is derived from spark.executor.memory (executorOverheadSettings gives the pair it is derived from),
+// spark.dynamicAllocation.initialExecutors from minExecutors,
+// and spark.sql.adaptive.autoBroadcastJoinThreshold falls back to spark.sql.autoBroadcastJoinThreshold
+// when unset. The lookup returns undefined for them rather than guess.
+
+import { parseSparkMemoryMB } from './spark-memory.js';
+
+                                                     
+
+                            
+                
+                     
+ 
+
+                             
+                                                                                                   
+                               
+                                    
+                                                       
+                                               
+                                                     
+ 
+
+                                                     
+                                                                     
+
+const ALWAYS = 'always';
+
+// The per-query (`modifiedConfigs`) keys a detector reads: every key with a default above, and the
+// two SQL settings it reads with none. A per-query value under any other key is the job's own text
+// (a path, a bucket, a host) that no detector needs.
+const PER_QUERY_KEYS_WITHOUT_DEFAULT = [
+  'spark.sql.adaptive.autoBroadcastJoinThreshold',
+  'spark.sql.adaptive.coalescePartitions.initialPartitionNum',
+]         ;
+export function isDetectorConfKey(key        )          {
+  return key in SPARK_DEFAULTS || (PER_QUERY_KEYS_WITHOUT_DEFAULT                     ).includes(key);
+}
+
+/** The text a per-query value is replaced with once it leaves the run: Spark's own placeholder for a
+ * hidden value, which the effective-conf lookup already ignores. */
+export const REDACTED_CONF_VALUE = '*********(redacted)';
+
+// Defaults by the Spark release that introduced them (table: detector-contract/threshold-tables.md,
+// Spark defaults by release), each key's steps in ascending order. A key whose first step is
+// 'always' has that default on every release, including a run whose version the log did not
+// record; any other key is only answered for a known version at or after its first step (before
+// it the property does not exist). Values are the strings Spark's ConfigBuilder declares. Each was read from the SQLConf.scala / config/package.scala sources at the release tags
+// from v3.0.0 through v4.2.0, e.g. https://github.com/apache/spark/blob/v4.0.0/core/src/main/scala/org/apache/spark/internal/config/package.scala
+const SPARK_DEFAULTS                                                   = {
+  'spark.speculation': [[ALWAYS, 'false']],
+  'spark.speculation.multiplier': [[ALWAYS, '1.5'], [[4, 0], '3']],
+  'spark.speculation.quantile': [[ALWAYS, '0.75'], [[4, 0], '0.9']],
+
+  'spark.executor.memory': [[ALWAYS, '1g']],
+  'spark.serializer': [[ALWAYS, 'org.apache.spark.serializer.JavaSerializer']],
+  'spark.shuffle.service.enabled': [[ALWAYS, 'false']],
+  'spark.eventLog.logBlockUpdates.enabled': [[ALWAYS, 'false']],
+  'spark.eventLog.logStageExecutorMetrics': [[ALWAYS, 'false']],
+
+  'spark.dynamicAllocation.enabled': [[ALWAYS, 'false']],
+  'spark.dynamicAllocation.minExecutors': [[ALWAYS, '0']],
+  'spark.dynamicAllocation.maxExecutors': [[ALWAYS, '2147483647']],
+  'spark.dynamicAllocation.executorIdleTimeout': [[ALWAYS, '60s']],
+
+  'spark.sql.shuffle.partitions': [[ALWAYS, '200']],
+  'spark.sql.autoBroadcastJoinThreshold': [[ALWAYS, '10MB']],
+  'spark.sql.files.maxPartitionBytes': [[ALWAYS, '128MB']],
+  'spark.sql.adaptive.enabled': [[ALWAYS, 'false'], [[3, 2], 'true']],
+  'spark.sql.adaptive.skewJoin.enabled': [[[3, 0], 'true']],
+  'spark.sql.adaptive.coalescePartitions.enabled': [[[3, 0], 'true']],
+  'spark.sql.adaptive.skewJoin.skewedPartitionFactor': [[[3, 0], '5']],
+  'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes': [[[3, 0], '256MB']],
+  'spark.sql.adaptive.forceOptimizeSkewedJoin': [[[3, 3], 'false']],
+  // Falls back to spark.sql.adaptive.shuffle.targetPostShuffleInputSize, itself 64MB.
+  'spark.sql.adaptive.advisoryPartitionSizeInBytes': [[[3, 0], '64MB']],
+  'spark.sql.adaptive.coalescePartitions.parallelismFirst': [[[3, 2], 'true']],
+  // The factor is 0.4 instead on Kubernetes non-JVM jobs, which the log does not tell apart.
+  'spark.executor.memoryOverheadFactor': [[[3, 3], '0.1']],
+  'spark.executor.minMemoryOverhead': [[[4, 0], '384m']],
+  'spark.sql.execution.pythonUDF.arrow.enabled': [[[3, 4], 'false'], [[4, 2], 'true']],
+};
+
+// The text Spark substitutes for a sensitive value in modifiedConfigs (a fixed constant; only the
+// pattern that selects values is configurable).
+const DEFAULT_REDACTION = REDACTED_CONF_VALUE;
+
+function parseVersion(sparkVersion                           )               {
+  const m = /^(\d+)\.(\d+)/.exec(sparkVersion ?? '');
+  return m == null ? null : [Number(m[1]), Number(m[2])];
+}
+
+/** Spark's default for `key` on `sparkVersion`; undefined when the property has no fixed default,
+ * does not exist on that version, or the default depends on a version the log did not record. */
+export function sparkConfDefault(sparkVersion                           , key        )                     {
+  const steps = SPARK_DEFAULTS[key];
+  if (steps == null) return undefined;
+  const version = parseVersion(sparkVersion);
+  if (version == null) return steps.length === 1 && steps[0][0] === ALWAYS ? steps[0][1] : undefined;
+  let value                    ;
+  for (const [since, stepValue] of steps) {
+    if (since === ALWAYS || version[0] > since[0] || (version[0] === since[0] && version[1] >= since[1])) value = stepValue;
+  }
+  return value;
+}
+
+/** A per-query setting that can override: not Spark's redaction placeholder for a hidden value. */
+function isUsableModified(value        )          {
+  return value !== DEFAULT_REDACTION;
+}
+
+/** The Spark Properties with one execution's modifiedConfigs applied over them. Returns
+ * `properties` itself when there is nothing to apply. */
+export function overlayModifiedConfigs(
+  properties                                    ,
+  modified                                                     ,
+)                                     {
+  if (modified == null) return properties;
+  const usable = Object.entries(modified).filter(([, value]) => isUsableModified(value));
+  return usable.length === 0 ? properties : { ...properties, ...Object.fromEntries(usable) };
+}
+
+/** The run's effective value of `key` and the layer it came from, or undefined when no layer has it. */
+export function effectiveSparkConf(layers            , key        )                        {
+  const modified = layers.modified?.[key];
+  if (modified !== undefined && isUsableModified(modified)) return { value: modified.trim(), source: 'query' };
+  const logged = layers.properties?.[key];
+  // A log can carry a real boolean or number where Spark wrote a string.
+  if (logged !== undefined) return { value: String(logged).trim(), source: 'app' };
+  const fallback = sparkConfDefault(layers.sparkVersion, key);
+  return fallback === undefined ? undefined : { value: fallback, source: 'default' };
+}
+
+/** A Spark byte-size value as bytes (a plain number, or with a k/m/g/t suffix, optionally 'b'), or
+ * null when it is not one. Negative numbers pass through: -1 turns auto-broadcast off. */
+export function parseSparkBytes(value                    )                {
+  const m = /^(-?\d+(?:\.\d+)?)\s*([kmgt]?)b?$/i.exec(value?.trim() ?? '');
+  if (m == null) return null;
+  return m[2] === '' ? Number(m[1]) : Number(m[1]) * 1024 ** ('kmgt'.indexOf(m[2].toLowerCase()) + 1);
+}
+
+/** The factor and minimum (MiB) Spark sizes the executor overhead from when
+ * spark.executor.memoryOverhead is unset: max(factor x executor memory, minimum). The effective
+ * settings win, each only on a Spark version that reads it (the factor from 3.3.0, the minimum from
+ * 4.0.0); an unrecorded version is read as current. `fallback` is the pair used where the effective
+ * conf has none (a version before the key, or an unreadable value). */
+export function executorOverheadSettings(
+  layers            , fallback                                       ,
+)                                    {
+  const version = parseVersion(layers.sparkVersion);
+  const [major, minor] = version ?? [Infinity, 0];
+  const factor = Number.parseFloat(effectiveSparkConf(layers, 'spark.executor.memoryOverheadFactor')?.value ?? '');
+  const minMB = parseSparkMemoryMB(effectiveSparkConf(layers, 'spark.executor.minMemoryOverhead')?.value);
+  return {
+    factor: (major > 3 || (major === 3 && minor >= 3)) && Number.isFinite(factor) && factor > 0 ? factor : fallback.floorPct,
+    minMB: major >= 4 && minMB != null && minMB >= 0 ? minMB : fallback.floorMB,
+  };
+}

package/vendor-core/sql-stages.js

@@ -1,5 +1,5 @@
-// Shared by every scope:'sql' detector and the plan views. sql.get(id).stageIds is always empty (parser-worker
-// never populates it), so stage linkage is derived from each stage's own sqlExecutionId.
+// Shared by every scope:'sql' detector and the plan views. sql.get(id).stageIds is always empty
+// (see detector-contract/plan-attribution.md), so stages are linked by their own sqlExecutionId.
 // Minimal param shape (not DetectorStage): external callers pass Map<StageId, Stage>.
 export function stageIdsForSqlExec(
   executionId        ,

package/vendor-core/stage-alignment.js

@@ -16,7 +16,7 @@ import { withEarlierAttempts, totalExecutorCpuMs } from './run-totals.js';
 import { planNodesOfStage } from './stage-plan-nodes.js';
 import { normalizeStageName, stageIdentityWith,                       } from './stage-identity.js';
 import { alignSequences } from './sequence-alignment.js';
-import { attributeNames, detailTokens, jaccard, structuralKey } from './stage-structure.js';
+import { attributeNamesOf, detailTokens, jaccard, structuralKey } from './stage-structure.js';
                                                              
                                                   
 
@@ -24,8 +24,6 @@ import { attributeNames, detailTokens, jaccard, structuralKey } from './stage-st
  * `runtimeCoverage`). Additive changes keep it; a consumer refuses a version it does not know. */
 export const COMPARISON_SCHEMA_VERSION = 1;
 
-// ---- Comparison normalizer ------------------------------------------------------------------
-
 // Targeted patches for the volatile tokens that differ between two runs of one job. Each applies
 // on top of `normalizeDetail`, in this order. No blanket numeric or path stripping: those fuse
 // distinct stages of one run.
@@ -95,8 +93,6 @@ export function comparisonDetailNormalizer(callerPatterns                    = [
   };
 }
 
-// ---- Identity and bookkeeping ---------------------------------------------------------------
-
 /** `stageIdentity`'s recipe (normalized stage name plus the attributed nodes' fingerprint) with the
  * comparison normalizer. Internal to the aligner: consumers key on a pair's `pairId`. */
 export function comparisonIdentity(
@@ -125,8 +121,6 @@ export function isDeltaBookkeepingStage(
   return scans > 0;
 }
 
-// ---- Alignment ------------------------------------------------------------------------------
-
 export const PAIR_DELTA_METRICS = [
   'executorRunTime', 'executorCpuTime', 'memoryBytesSpilled', 'diskBytesSpilled',
   'inputBytes', 'outputBytes', 'shuffleReadBytes', 'shuffleWriteBytes',
@@ -268,8 +262,6 @@ function sumFigures(rows           )          {
 
 const ascending = (a        , b        )         => a - b;
 
-// ---- Per-run index --------------------------------------------------------------------------
-
 // What the aligner knows about one SQL execution (or the pseudo-execution of stages outside any):
 // the non-bookkeeping stages it ran and the three things its similarity reads.
                     
@@ -325,7 +317,7 @@ function planStructure(root          )           {
   (function walk(node          )       {
     if (!/^(WholeStageCodegen|InputAdapter|ColumnarToRow)\b/.test(node.name ?? '')) {
       tokens.push(normalizeStageName(node.name ?? ''));
-      for (const name of attributeNames(node.detail ?? '')) attributes.add(`@${name}`);
+      for (const name of attributeNamesOf(node)) attributes.add(`@${name}`);
     }
     for (const child of node.children ?? []) walk(child);
   })(root);
@@ -395,8 +387,6 @@ function executionSimilarity(a          , b          )         {
   return Math.round((1000 * sum) / weight);
 }
 
-// ---- Stage alignment inside one execution pair -----------------------------------------------
-
                                                                                                   
 
 function alignWithin(

package/vendor-core/stage-identity.js

@@ -13,6 +13,24 @@ import { planNodesOfStage } from './stage-plan-nodes.js';
 // Replace run-varying tokens (digit runs, long hex ids) with a stable marker so
 // the same logical stage across two runs normalizes to one identity.
 export function normalizeStageName(name        )         {
+  if (name.length > STAGE_NAME_CACHE_MAX_LENGTH) return computeStageName(name);
+  let normalized = stageNameCache.get(name);
+  if (normalized === undefined) {
+    if (stageNameCache.size >= STAGE_NAME_CACHE_MAX_ENTRIES) stageNameCache.clear();
+    normalized = computeStageName(name);
+    stageNameCache.set(name, normalized);
+  }
+  return normalized;
+}
+
+// A comparison normalizes the same few operator and stage names once per stage, identity and key.
+// Only short names are kept (long inputs are call-site text, rarely repeated), and the whole table
+// resets when full, so it stays small however many runs a long-lived process analyzes.
+const STAGE_NAME_CACHE_MAX_LENGTH = 256;
+const STAGE_NAME_CACHE_MAX_ENTRIES = 4096;
+const stageNameCache = new Map                ();
+
+function computeStageName(name        )         {
   return String(name)
     .toLowerCase()
     .replace(/\b[0-9a-f]{8,}\b/g, '#') // hex ids/uuids first (they contain digits)
@@ -21,6 +39,28 @@ export function normalizeStageName(name        )         {
     .trim();
 }
 
+// A plan node belongs to every stage that ran part of it, and each of those stages reads its
+// normalized detail again (exact key, structural key, similarity tokens), on both runs of a
+// comparison. The normalizer is a pure function of the detail and a node is never mutated after the
+// parser posts it, so the result is memoized per node. Like the per-tree identity below, each node
+// keeps at most MAX_NORMALIZERS_PER_TREE normalizers and drops the oldest first.
+const nodeDetailCache = new WeakMap                                         ();
+
+export function normalizedDetailOf(node          , normalizeDetail                  )         {
+  let byNormalizer = nodeDetailCache.get(node);
+  if (!byNormalizer) {
+    byNormalizer = new Map();
+    nodeDetailCache.set(node, byNormalizer);
+  }
+  let normalized = byNormalizer.get(normalizeDetail);
+  if (normalized === undefined) {
+    normalized = normalizeDetail(node.detail ?? '');
+    if (byNormalizer.size >= MAX_NORMALIZERS_PER_TREE) byNormalizer.delete(byNormalizer.keys().next().value );
+    byNormalizer.set(normalizeDetail, normalized);
+  }
+  return normalized;
+}
+
 // Bottom-up, order-independent structural identity of a resolved plan tree:
 // each node folds its normalized name/detail with its children's digests
 // (children sorted, so AQE picking a different broadcast side still matches),
@@ -32,13 +72,28 @@ export function normalizeStageName(name        )         {
 // instead of embedding full child identity strings, which would re-escape
 // every level below and blow identity size up to ~2^depth on the 20-60+
 // operator-deep plans real Spark produces.
+//
+// Memoized per resolved tree (a tree is never mutated after the parser posts it), because every
+// stage of an execution that has no attributed node falls back to the same whole-tree identity.
+// The aligner builds a fresh normalizer closure per call, so each tree keeps at most
+// MAX_NORMALIZERS_PER_TREE entries and drops the oldest first.
+const MAX_NORMALIZERS_PER_TREE = 4;
+const planTreeIdentityCache = new WeakMap                                         ();
+
 function planTreeIdentity(root                             , normalizeDetail                  )                {
   if (!root) return null;
+  const byNormalizer = planTreeIdentityCache.get(root) ?? new Map                          ();
+  const cached = byNormalizer.get(normalizeDetail);
+  if (cached !== undefined) return cached;
   function visit(node          )         {
     const childDigests = (node.children ?? []).map(visit).sort();
-    return cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? ''), childDigests]));
+    return cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), normalizedDetailOf(node, normalizeDetail), childDigests]));
   }
-  return visit(root);
+  const identity = visit(root);
+  if (byNormalizer.size >= MAX_NORMALIZERS_PER_TREE) byNormalizer.delete(byNormalizer.keys().next().value );
+  byNormalizer.set(normalizeDetail, identity);
+  planTreeIdentityCache.set(root, byNormalizer);
+  return identity;
 }
 
 // Plan identity for the stage's SQL execution, scoped to only the plan nodes
@@ -56,7 +111,7 @@ function sqlNodeIdentity(
   const root = snapshot.sql.get(execId)?.planTree ?? null;
   if (!root) return '';
   const fingerprints = (nodes ?? planNodesOfStage(stage, snapshot.sql))
-    .map((node) => JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
+    .map((node) => JSON.stringify([normalizeStageName(node.name ?? ''), normalizedDetailOf(node, normalizeDetail)]));
   if (fingerprints.length === 0) return planTreeIdentity(root, normalizeDetail) ?? '';
   return cyrb53(JSON.stringify(fingerprints.sort()));
 }

package/vendor-core/stage-label.js

@@ -0,0 +1,14 @@
+// A path inside a stage name: an absolute path, optionally behind a URI scheme and authority
+// (`hdfs://nn/...`). It starts at the beginning of the name or after a space, quote, bracket, `=`
+// or `,`, so a `/` inside a word ("a/b") is never taken for one. A `:` ends the path, which leaves
+// a trailing `:<line>` in place.
+const PATH_IN_STAGE_NAME = /(?<![^\s"'(\[=,])(?:[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s/]*)?(?:\/[^\s/:"')\]]+)+/g;
+
+/** A stage name for display and output: every path in it reduced to its last segment, so the
+ * directories of the job's call site (a YARN container directory such as
+ * `.../usercache/<user>/appcache/<app>/container_<id>/`, a home directory) and the user name in
+ * them never reach the output. `collect at /home/jdoe/jobs/etl.py:42` reads `collect at etl.py:42`.
+ * Display only: `normalizeStageName` stays the stage identity, so stage pairing does not move. */
+export function stageDisplayLabel(name        )         {
+  return name.replace(PATH_IN_STAGE_NAME, (path) => path.slice(path.lastIndexOf('/') + 1));
+}

package/vendor-core/stage-plan-nodes.js

@@ -9,10 +9,24 @@ export function planNodesOfStage(stage                                      , sq
   if (execId == null) return [];
   const root = sql.get(execId)?.planTree ?? null;
   if (!root) return [];
-  const nodes             = [];
+  return [...(nodesByStage(root).get(stage.id) ?? [])];
+}
+
+// One preorder walk per resolved plan tree serves every stage of its execution. A resolved tree is
+// never mutated after the parser posts it, so keying by root identity is safe.
+const nodesByStageCache = new WeakMap                                   ();
+
+function nodesByStage(root          )                          {
+  const cached = nodesByStageCache.get(root);
+  if (cached) return cached;
+  const index = new Map                    ();
   (function collect(node          )       {
-    if (node.stageIds?.includes(stage.id)) nodes.push(node);
+    for (const id of new Set(node.stageIds ?? [])) {
+      const list = index.get(id);
+      if (list) list.push(node); else index.set(id, [node]);
+    }
     for (const child of node.children ?? []) collect(child);
   })(root);
-  return nodes;
+  nodesByStageCache.set(root, index);
+  return index;
 }