sparkforensics-server 0.4.0 → 0.5.1

package/vendor-core/impact-model.js

@@ -1,7 +1,8 @@
 // The waste models every detector entry's estimate() builds its ImpactEstimate from: the assumed
 // throughputs, the per-stage measurements behind them and the occupancy clip wrappers. Each
 // finding type's own composition of these lives on its DETECTORS entry, next to its detect().
-                                                                                              
+                                                                                                                     
+import { isPythonStage } from './python-stage.js';
 import { nsToMs } from './format-utils.js';
 import {
   estimateSingleStage, estimateMultiStage,
@@ -16,6 +17,8 @@ import {
                                              
                                                                                  
                      
+                                                                                                  
+                                  
  
 
 // Assumed shuffle-network throughput per executor link, ~1 Gbps. Starting assumption, unvalidated.
@@ -104,13 +107,14 @@ const IDLE_CPU_SHARE_MAX = 0.01;
 
 // True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
 // the share can't be trusted: no CPU time recorded (older Spark logs omit the metric), or Python
-// code run through PythonRDD, whose worker-process CPU executorCpuTime (the JVM task thread's)
-// never counts (such stages read 0.1% on the same logs while computing).
-export function tasksMostlyIdle(stage       )          {
+// code run through a Python worker (isPythonStage: a PythonRDD stage or a Python UDF operator in
+// its SQL plan), whose worker-process CPU executorCpuTime (the JVM task thread's) never counts
+// (such stages read 0.1% on the same logs while computing).
+export function tasksMostlyIdle(stage       , sql                            = new Map())          {
   const runMs = stage.executorRunTime ?? 0;
   const cpuMs = nsToMs(stage.executorCpuTime ?? 0);
   if (runMs <= 0 || cpuMs <= 0) return false;
-  if (/PythonRDD/.test(stage.name ?? '') || /org\.apache\.spark\.api\.python\./.test(stage.details ?? '')) return false;
+  if (isPythonStage(stage, sql)) return false;
   return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
 }
 
@@ -173,3 +177,21 @@ export function stageMappableWasteOrCostOnly(
   // null: every stage excluded from the sweep
   return multiStageImpact(stageIds, wasteMsByStage, ctx, 'modeled', rawWaste) ?? costOnly('modeled', rawWaste);
 }
+
+// Finding types whose raw figure is busy core time read straight from the log: gc's jvmGCTime
+// (coreMs) and the discarded speculative or retried attempts' run time ('ms' cross-task sums).
+// Any other core figure is idle capacity, or modeled on an assumed constant (coreLocality's
+// per-task fetch penalty, autoscalingChurn's executor-hours, jobFailureRate's job-hours).
+const MEASURED_CORE_TIME_FIGURE                      = new Set(['gc', 'retryWaste', 'speculationWaste']);
+
+/** The busy core time a finding's fix removes, in core-milliseconds, or null when the detector
+ * measures none. Only a measured figure counts: one its estimate() already set (skew and
+ * straggler's removed task time), or the raw figure of a MEASURED_CORE_TIME_FIGURE type. A
+ * wall-clock claim, an idle capacity figure and a modeled figure are never converted. It never
+ * reads executorCpuTime, which leaves out Python worker CPU. */
+export function coreTimeFor(finding         , estimate                )                                       {
+  if (estimate.coreTimeMs !== undefined) return estimate.coreTimeMs;
+  const raw = estimate.rawWaste;
+  if (!raw || !MEASURED_CORE_TIME_FIGURE.has(finding.type)) return null;
+  return { low: raw.value, high: raw.value };
+}

package/vendor-core/ingest.js

@@ -14,6 +14,7 @@
                                             
                                                    
                                                     
+                                                   
                                    
                                     
  
@@ -39,9 +40,10 @@ export function routeMessage(
     case 'runAggregates': handlers.onRunAggregates?.(data.data); break;
     case 'stageExecutorMetrics': handlers.onStageExecutorMetrics?.(data.data); break;
     case 'stageSpeculationWaste': handlers.onStageSpeculationWaste?.(data.data); break;
+    case 'stageLateAttemptWork': handlers.onStageLateAttemptWork?.(data.data); break;
     case 'done': {
-      const { skippedLines } = data;
-      handlers.onDone?.({ skippedLines });
+      const { skippedLines, unreadableSqlExecutions } = data;
+      handlers.onDone?.(unreadableSqlExecutions ? { skippedLines, unreadableSqlExecutions } : { skippedLines });
       break;
     }
     case 'error':    handlers.onError?.(data); break;

package/vendor-core/list-runs.js

@@ -5,6 +5,7 @@ import { reassembleRollingEntries } from './parser-worker.js';
 import { peekLogHeader } from './log-header-peek.js';
 import { mcpError } from './mcp-error.js';
 import { normalizeBaseUrl } from './shs-request.js';
+import { isConnectionFailure } from './proxy.js';
 import { DEFAULT_IDLE_TIMEOUT_MS, DEFAULT_MAX_ARCHIVE_BYTES } from './shs-load.js';
 
                                
@@ -205,10 +206,12 @@ export async function listRunsShs(
   let res          ;
   try {
     // An unresponsive SHS would otherwise hang the tool call forever. A timed-out signal rejects
-    // the fetch with an AbortError, which this same catch turns into access-or-upstream-failure.
+    // the fetch with a TimeoutError, which this same catch turns into upstream-unreachable, the
+    // code a refused connection gets too.
     res = await fetchImpl(url.toString(), { signal: AbortSignal.timeout(DEFAULT_IDLE_TIMEOUT_MS) });
   } catch (e) {
-    throw mcpError('access-or-upstream-failure', `Could not reach ${normalized}: ${e instanceof Error ? e.message : String(e)}`);
+    const code = isConnectionFailure(e) ? 'upstream-unreachable' : 'access-or-upstream-failure';
+    throw mcpError(code, `Could not reach ${normalized}: ${e instanceof Error ? e.message : String(e)}`);
   }
   if (!res.ok) {
     throw mcpError('access-or-upstream-failure', `SHS applications list request failed with status ${res.status}.`);

package/vendor-core/mcp-server-factory.js

@@ -6,6 +6,7 @@ import {
 } from './mcp-tools.js';
 import { listRuns } from './list-runs.js';
                                                          
+import { MAX_NORMALIZE_PATTERN_LENGTH, MAX_NORMALIZE_PATTERNS } from './stage-alignment.js';
 
 const sourceSchema = z.union([
   z.object({ path: z.string() }),
@@ -71,7 +72,7 @@ export function createMcpServer({ thresholds }
   }, (params) => toolResult(listRuns(params)));
 
   server.registerTool('diagnose_run', {
-    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, and the checks the log lacked the data to run. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
+    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, the checks the log lacked the data to run, and the writeTargets, metrics and effectiveConf blocks of the CLI report. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
     inputSchema: {
       ...runRefSchema, redact: z.boolean().optional(),
       include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
@@ -91,15 +92,19 @@ export function createMcpServer({ thresholds }
   ));
 
   server.registerTool('compare_runs', {
-    description: 'Compare two runs: categorized findings delta and metric deltas.',
+    description: 'Compare two runs: the verdict, metric deltas (metrics), categorized findings delta (findings) and the stage pairs (stagePairs) with the share of executor run time they cover (runtimeCoverage). confidence is ok, low or insufficient. metricDeltas and findingsDelta repeat metrics and findings under their earlier names and will be removed.',
     inputSchema: {
       runIdA: z.string().optional(), sourceA: sourceSchema.optional(),
       ...secondRunRefSchema,
       redact: z.boolean().optional(),
+      include: z.array(z.enum(['stagePairs'])).optional()
+        .describe('Extra blocks to return. stagePairs: one row per paired stage with its deltas, which can be large.'),
+      normalizePath: z.array(z.string().max(MAX_NORMALIZE_PATTERN_LENGTH)).max(MAX_NORMALIZE_PATTERNS).optional()
+        .describe('Regular expressions whose every match in a plan node\'s text is replaced with a fixed token before stages are paired, so run-specific text (an output directory that differs per run) does not keep the same stage from pairing. A generic pattern, not only for paths. Affects stage pairing, runtimeCoverage and confidence, never findings. A pattern that backtracks catastrophically can stall the server: keep them simple and anchored.'),
       ...formatSchema,
     },
-  }, ({ runIdA, sourceA, runIdB, sourceB, redact, format }) => toolResultWithMarkdown(
-    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds }),
+  }, ({ runIdA, sourceA, runIdB, sourceB, redact, include, normalizePath, format }) => toolResultWithMarkdown(
+    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds, normalizePath, include }),
   ));
 
   server.registerTool('evaluate_budgets', {
@@ -112,14 +117,16 @@ export function createMcpServer({ thresholds }
       sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced). When given, the absolute budgets (maxRuntimeMs etc.) are evaluated on this run.'),
       runIdB: secondRunRefSchema.runIdB.describe('Same as `sourceB`, referencing an already-resolved run by id.'),
       maxRegressionPct: z.number().optional(), regressionMetric: z.string().optional(), failOnIntroduced: z.string().optional(),
+      regressionBudgets: z.array(z.object({ metric: z.string(), maxPct: z.number().nonnegative() })).optional()
+        .describe('Further regression budgets, one per metric (the CLI\'s repeated --regression-budget). A metric can be budgeted once across this list and maxRegressionPct/regressionMetric.'),
     },
   }, ({
     source, runId, runIdB, sourceB,
     maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct,
-    maxRegressionPct, regressionMetric, failOnIntroduced,
+    maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets,
   }) => toolResult(evaluateBudgetsForRun(
     { source, runId },
-    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced },
+    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets },
     (runIdB || sourceB) ? { runId: runIdB, source: sourceB } : undefined,
     { thresholds },
   )));

package/vendor-core/mcp-tools.js

@@ -6,13 +6,14 @@ import { collectRun } from './cli/collect-run.js';
 import { deriveEvidenceAvailability } from './evidence-availability.js';
 import { resolveFromShs, DEFAULT_MAX_ARCHIVE_BYTES, DEFAULT_IDLE_TIMEOUT_MS } from './shs-load.js';
 import { mcpError } from './mcp-error.js';
-import { buildEvidenceReport, toFindingsFilter,                                                                                                               } from './evidence-report.js';
-import { redactComparison } from './redact.js';
+import { buildEvidenceReport, runFindings, toFindingsFilter,                                                                                                               } from './evidence-report.js';
 import { computeWallClock } from './wall-clock.js';
-import { analyze } from './analyzer.js';
-import { buildComparison, renderComparisonMarkdown,                        } from './run-comparison.js';
-import { comparisonVerdict,                            } from './comparison-verdict.js';
+import { renderComparisonMarkdown,                        } from './run-comparison.js';
+import { buildComparisonOutput,                       } from './comparison-output.js';
+import { compileNormalizePatterns } from './stage-alignment.js';
+import { runOutputBlocks } from './run-output.js';
 import { evaluateBudgets,                                       } from './cli/budgets.js';
+import { combineRegressionBudgets } from './cli/regression-budgets.js';
 import { FINDING_NAMES, titleCase } from './finding-names.js';
 import { docAnchorForType } from './detector-docs.js';
 import { DETECTORS,                         } from './detectors.js';
@@ -46,19 +47,15 @@ import { typeTag } from './format-utils.js';
                                                                                                
                      
  
-// compareRuns returns a smaller MCP-facing projection of CompareRunsResult
-// (runIdA/runIdB/verdict/findingsDelta/metricDeltas/confidence/reason/matchedCoverage), not the full
-// raw shape (no baselineLabel/stageSkew/baseStages/candStages/jobOutcomes).
-                                       
+// compareRuns returns the shared comparison projection (comparison-output.ts, the CLI's `comparison`
+// object), not the full raw shape (no baselineLabel/stageSkew/baseStages/candStages/jobOutcomes).
+// `verdict` is the dashboard comparison page's headline (baseline = runIdA, candidate = runIdB).
+                                                                
                  
                  
                                                                                
-                                 
-                                               
-                                             
                                               
-                                      
-                                                        
+                                            
  
 
 function envInt(name        , fallback        )         {
@@ -192,6 +189,7 @@ export function diagnoseRun(runId        , opts
                                                                                                       
                                                                                                          
                                                     
+                                                                                 
                                                                                                                 {
   const appModel = getCachedAppModel(runId);
   const findingsFilter = toFindingsFilter(opts?.impactBand, opts?.type, opts?.stageId);
@@ -203,6 +201,9 @@ export function diagnoseRun(runId        , opts
   return {
     runId, verdict: json.verdict, findings: json.findings, recommendations: json.recommendations, cleanChecks: json.cleanChecks,
     notRunChecks: json.notRunChecks,
+    // The CLI report's own blocks, so a run reads the same through either surface.
+    writeTargets: json.writeTargets,
+    ...runOutputBlocks(appModel, { redact: opts?.redact, thresholds: opts?.thresholds }),
     runComplete: appModel.app?.endTime != null,
     // Top level too, so a client that never asks for `summary` still sees the run was tuned.
     ...(tuned ? { tunedThresholds: tuned } : {}),
@@ -345,16 +346,19 @@ async function resolveAndAnalyze(
   ref        , thresholds                                ,
 )                                                                     {
   const { runId, appModel } = await resolveOrCreateRun(ref);
-  const catalog = analyze(
-    appModel.app, appModel.stages, appModel.executors.added, appModel.executors.removed,
-    appModel.jobs, appModel.sql, appModel.runAggregates, { thresholds },
-  );
-  return { runId, appModel, catalog };
+  return { runId, appModel, catalog: runFindings(appModel, thresholds).catalog };
 }
 
 export async function compareRuns(
-  a        , b        , opts                                                                            ,
+  a        , b        , opts                                                                                                              
+                                                                                                                  ,
 )                                                                                                           {
+  // Before any log is read: a bad pattern refuses the call, as the CLI's usage error does.
+  try {
+    compileNormalizePatterns(opts?.normalizePath);
+  } catch (e) {
+    throw mcpError('access-or-upstream-failure', `normalizePath: ${(e         ).message}`);
+  }
   const [
     { runId: runIdA, appModel: appModelA, catalog: catalogA },
     { runId: runIdB, appModel: appModelB, catalog: catalogB },
@@ -364,26 +368,20 @@ export async function compareRuns(
   // interactive stage-detail drill-down, which none of compare/matchStages/metricDeltas/findingsDelta
   // read. Findings/metrics come from `catalog` and appModel.stages/sql, both fully populated. Produces
   // identical output to the dashboard; the MCP tool just never exposes per-task drill-down.
-  const built = buildComparison(
+  const { comparison, output } = buildComparisonOutput(
     { label: runIdA, appModel: appModelA, catalog: catalogA },
     { label: runIdB, appModel: appModelB, catalog: catalogB },
+    { redact: opts?.redact, normalizePath: opts?.normalizePath, view: opts?.include?.includes('stagePairs') ? 'full' : 'summary' },
   );
-  // Stage names throughout `built` carry raw Spark stage text, which can embed a host/IP token as
-  // free text, the same residual redactReport() already scrubs from the evidence report.
-  const result = opts?.redact ? redactComparison(built) : built;
-  const verdict = comparisonVerdict(result);
 
   return {
     runIdA,
     runIdB,
-    verdict,
-    findingsDelta: result.findings,
-    metricDeltas: result.metrics,
-    confidence: result.confidence,
-    reason: result.reason,
-    matchedCoverage: result.matchedCoverage,
+    ...output,
+    findingsDelta: output.findings,
+    metricDeltas: output.metrics,
     ...tunedField(opts?.thresholds),
-    ...(opts?.markdown ? { markdown: renderComparisonMarkdown(result, verdict, tunedDetectors(opts?.thresholds)) } : {}),
+    ...(opts?.markdown ? { markdown: renderComparisonMarkdown(comparison, output.verdict, tunedDetectors(opts?.thresholds)) } : {}),
   };
 }
 
@@ -408,6 +406,16 @@ export async function evaluateBudgetsForRun(
   if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
     throw mcpError('access-or-upstream-failure', 'regressionMetric requires maxRegressionPct.');
   }
+  // The CLI's rule: a metric is budgeted once across the single pair and the list.
+  try {
+    const pair = budgets.maxRegressionPct === undefined ? [] : [{ metric: budgets.regressionMetric ?? 'wallClock', maxPct: budgets.maxRegressionPct }];
+    combineRegressionBudgets([
+      ...pair.map((budget) => ({ origin: 'maxRegressionPct/regressionMetric', budget })),
+      ...(budgets.regressionBudgets ?? []).map((budget) => ({ origin: 'regressionBudgets', budget })),
+    ]);
+  } catch (e) {
+    throw mcpError('access-or-upstream-failure', (e         ).message);
+  }
   const [first, second] = await Promise.all([
     resolveAndAnalyze(primary, opts?.thresholds),
     secondary ? resolveAndAnalyze(secondary, opts?.thresholds) : Promise.resolve(undefined),
@@ -418,11 +426,12 @@ export async function evaluateBudgetsForRun(
   // apply to the candidate.
   const candidate = second ?? first;
   const baseline = second ? first : undefined;
+  // Not redacted: no budget reads the stage text redaction rewrites (the parity suite checks it).
   const comparison                                = baseline
-    ? buildComparison(
+    ? buildComparisonOutput(
       { label: baseline.runId, appModel: baseline.appModel, catalog: baseline.catalog },
       { label: candidate.runId, appModel: candidate.appModel, catalog: candidate.catalog },
-    )
+    ).comparison
     : undefined;
 
   const { results, violated, inconclusive } = evaluateBudgets({

package/vendor-core/model-assembler.js

@@ -2,6 +2,7 @@
            
                
         
+                     
                
            
                 
@@ -9,6 +10,15 @@
                 
                     
 
+// The done message's counts of what the parse could not read. On the model, not only passed to onDone,
+// so every consumer of the model (dashboard export, CLI, MCP) reports them the same way.
+function recordParseGaps(appModel          , data         )       {
+  const done = data                                                                             ;
+  appModel.skippedLines = done?.skippedLines ?? 0;
+  if (done?.unreadableSqlExecutions) appModel.unreadableSqlExecutions = done.unreadableSqlExecutions;
+  else delete appModel.unreadableSqlExecutions;
+}
+
 // Worker-message -> appModel assembly. Shared by the file-load and SHS-URL-load paths. Pure model
 // mutation: analysis/render/persist stay in the caller via the onDone/onProgress/onError hooks.
 export function createModelCallbacks(
@@ -75,6 +85,18 @@ export function createModelCallbacks(
         }
       }
     },
-    onDone, onError,
+    onDone(data         ) {
+      recordParseGaps(appModel, data);
+      onDone?.(data);
+    },
+    // Patch the late work of failed attempts, whose TaskEnds arrive after StageCompleted.
+    // `data` is Map<stageId, StageAttemptTotals>.
+    onStageLateAttemptWork(data         ) {
+      for (const [stageId, work] of data                                   ) {
+        const stage = appModel.stages.get(stageId);
+        if (stage) stage.lateAttemptWork = work;
+      }
+    },
+    onError,
   };
 }

package/vendor-core/parser-worker.js

@@ -45,7 +45,7 @@ const NOT_AN_EVENT_LOG = 'Not a Spark event log: no application-start event foun
 // Minimal shape streamFile/runParse/runParseFiles read off `file` (name, size,
 // slice(start,end).arrayBuffer()): narrower than the full DOM `File`. A real
 // `File` (the browser Worker path) satisfies it structurally, but so does the
-// plain object src/cli/collect-run.ts's nodeFileFromPath builds for Node, which
+// plain object packages/core/src/cli/collect-run.ts's nodeFileFromPath builds for Node, which
 // has no DOM `File` constructor.
                    
                

package/vendor-core/plan-graph-model.js

@@ -1,6 +1,6 @@
 // Flattens a resolved planTree into a { nodes, edges } graph shape for the
 // React Flow + Dagre plan graph view ("Plan graph view" in
-// docs-site/contributor-guide/architecture/drill-down.md). The Exchange
+// docs-site/contributor-guide/architecture/drill-down/plan-graph.md). The Exchange
 // write/read split is done upstream now, in resolvePlanTree
 // (event-handlers.ts): this file just walks the already-split tree and maps
 // each real PlanNode onto a PlanGraphNodeData. sourceNodeId still lets a

package/vendor-core/plan-summary.js

@@ -44,6 +44,17 @@ function pushLongFilterWarning(result             , len        )       {
   result.warnings.push({ type: 'longFilterCondition', detail: `Filter condition is ${len} characters: consider simplifying or pushing the filter down closer to the scan.` });
 }
 
+// True when a node reads internal Delta metadata: the _delta_log state scan or an anonymous
+// commit/checkpoint read. `Location:` is often truncated to `Del...`, so it also keys off
+// DeltaLogFileIndex or the Delta-log action column signature. (Column-signature heuristic: a
+// future Spark renaming those action columns would leak a noise row; acceptable.) Shared by
+// `scanRelationId` and the comparison's Delta bookkeeping flag (stage-alignment.ts).
+export function isDeltaLogRead(name        , detail        )          {
+  return /_delta_log/.test(name) || /_delta_log/.test(detail) ||
+    /DeltaLogFileIndex/i.test(name + detail) ||
+    /checkpointMetadata#|sidecar#|commitInfo#/.test(name + detail);
+}
+
 // Stable relation-identity key for a scan node: "<format>:<relation>" (e.g.
 // "delta:mx.store_map", "parquet:warehouse.sales", "jdbc:dw.dim_product")
 // or null when the node is internal Delta metadata / a non-scan / un-nameable.
@@ -70,16 +81,8 @@ export function scanRelationId(name        , detail        )                {
     return `${format}:${table.replace(/^spark_catalog\./, '')}`;
   }
 
-  // 2. Internal Delta metadata (the _delta_log state scan and anonymous
-  //    commit/checkpoint reads) → null. Location: is often truncated to `Del...`,
-  //    so also key off DeltaLogFileIndex or the Delta-log action column signature.
-  //    (Column-signature heuristic: a future Spark renaming those action columns
-  //    would leak a noise row; acceptable.)
-  if (/_delta_log/.test(name) || /_delta_log/.test(detail) ||
-      /DeltaLogFileIndex/i.test(name + detail) ||
-      /checkpointMetadata#|sidecar#|commitInfo#/.test(name + detail)) {
-    return null;
-  }
+  // 2. Internal Delta metadata (see isDeltaLogRead) → null.
+  if (isDeltaLogRead(name, detail)) return null;
 
   // 3. Anonymous real file read (InMemoryFileIndex, no catalog name) →
   //    best-effort Location basename. Preserves temp/snapshot self-reuse; a

package/vendor-core/proxy.js

@@ -10,7 +10,9 @@ function sendSafeError(res, status, code) {
   res.end(body);
 }
 
-function isConnectionFailure(error) {
+// Also read by list-runs.ts, so a History Server listing that can't be reached reports the
+// same upstream-unreachable code as a run fetch.
+export function isConnectionFailure(error) {
   if (error?.name === 'AbortError' || error?.name === 'TimeoutError') return true;
   const code = error?.code ?? error?.cause?.code;
   if (['ECONNREFUSED', 'ECONNRESET', 'EHOSTUNREACH', 'ENETUNREACH', 'ENOTFOUND', 'ETIMEDOUT'].includes(code)) return true;

package/vendor-core/python-stage.js

@@ -0,0 +1,25 @@
+import { planNodesOfStage } from './stage-plan-nodes.js';
+                                                      
+
+// Plan operators that hand rows to a Python worker process: the row-at-a-time and Arrow Python UDF
+// evaluators, the pandas/Arrow grouped and map operators, and the PythonRDD scan of an RDD
+// pipeline. A suffixed name is a variant of the same operator (BatchEvalPythonUDTF,
+// FlatMapGroupsInPandasWithState). Spark 4.1 renamed AggregateInPandas and WindowInPandas to
+// ArrowAggregatePython and ArrowWindowPython. Spark prefixes a whole-stage-codegen child's name
+// with "*(n) " in some plan strings.
+const PYTHON_PLAN_NODE = /^(?:\*\(\d+\)\s*)?(?:PythonRDD|BatchEvalPython|ArrowEvalPython|ArrowAggregatePython|ArrowWindowPython|\w+InPandas|\w+InArrow)\w*\b/;
+
+// An RDD lambda or map function has no SQL plan to match, and a stage whose plan could not be
+// matched is left with nothing but its name and call site.
+const PYTHON_STAGE_NAME = /PythonRDD/;
+const PYTHON_STAGE_DETAILS = /org\.apache\.spark\.api\.python\./;
+
+/** True when the stage ran Python code in a worker process: the union of a Python operator among
+ * the plan nodes attributed to it (catches Python UDFs inside SQL) and the stage's own name or
+ * call site naming PythonRDD / org.apache.spark.api.python (catches RDD lambdas, which have no
+ * plan, and stages that cannot be matched to one). The executor CPU time of such a stage misses
+ * the worker process's CPU. */
+export function isPythonStage(stage       , sql                           )          {
+  if (PYTHON_STAGE_NAME.test(stage.name ?? '') || PYTHON_STAGE_DETAILS.test(stage.details ?? '')) return true;
+  return planNodesOfStage(stage, sql).some((node) => PYTHON_PLAN_NODE.test(node.name ?? ''));
+}

package/vendor-core/recommendation-rollup.js

@@ -1,7 +1,7 @@
 // Explicit .ts extensions: plain Node's ESM resolver (the runtime CLI/MCP path
 // runs under) requires the exact specifier, unlike a bundler.
 import { mergeIntervals } from './intervals.js';
-import { formatWallClockRange, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
+import { formatRawWaste, formatWallClockRange, readsAsZero, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
                                                                     
 
                          
@@ -124,11 +124,15 @@ export function buildRecommendationRollup(
   });
 }
 
-/** A group's trailing figure on the Findings board, and its tooltip. `time`
- * and `resource` both lead with the "×N" finding count (the "worth
- * expanding" signal); `count` skips it since the impact-band tally already
- * implies N. The row stays terse ("×2 · 476ms recoverable") to fit a dense
- * right-aligned column; the title spells the shorthand out. */
+/** A resource group's summed waste as the board and CLI print it; null when it reads as zero. */
+export function resourceGroupTotal(group                                       )                {
+  const text = formatRawWaste({ value: group.total, unit: group.unit });
+  return readsAsZero(text) ? null : text;
+}
+
+/** A group's trailing figure on the Findings board, and its tooltip. Every kind leads with the
+ * "×N" finding count (the "worth expanding" signal). The row stays terse ("×2 · 476ms
+ * recoverable") to fit a dense right-aligned column; the title spells the shorthand out. */
 export function rollupGroupStat(group             )                                      {
   if (group.kind === 'time') {
     const recoverable = formatWallClockRange(group.recoverableMsHigh, group.recoverableMsHigh);
@@ -138,13 +142,16 @@ export function rollupGroupStat(group             )
     };
   }
   if (group.kind === 'resource') {
+    const total = resourceGroupTotal(group);
     return {
-      stat: `×${group.findingCount} · resource-cost projection`,
-      statTitle: `${group.findingCount} findings of this type; a resource-cost estimate (not run time) is projected for fixing them`,
+      stat: total ? `×${group.findingCount} · ${total}` : `×${group.findingCount}`,
+      statTitle: `${group.findingCount} findings of this type${total ? `; ${total} in total, a resource cost, not run time` : ''}`,
     };
   }
+  // The band heading above the row already names a one-band group's band.
+  const bands = Object.entries(group.byImpactBand);
   return {
-    stat: Object.entries(group.byImpactBand).map(([impactBand, count]) => `${count} ${impactBand}`).join(', '),
+    stat: bands.length === 1 ? `×${group.findingCount}` : `×${group.findingCount} · ${bands.map(([impactBand, count]) => `${count} ${impactBand}`).join(', ')}`,
     statTitle: `${group.findingCount} findings of this type, by impact`,
   };
 }

package/vendor-core/redact.js

@@ -9,13 +9,12 @@
 
 import { decodeCollections, encodeCollections,                    } from './export-data.js';
                                                     
-import { redactTaskFailureGroup,                        } from './task-failure.js';
+import { REDACTED_TEXT, redactTaskFailureGroup,                        } from './task-failure.js';
 
 // Host / IP identifier patterns. Used to enumerate host names that surface only
-// inside free text: recommendation strings, `stageFailed`'s failure-reason
-// value, SQL relation/node names, never as a structured `host` field, so
-// redaction reaches those residuals too. Pseudonyms (`host-1`) match neither
-// pattern, keeping the scan idempotent.
+// inside free text: recommendation strings, SQL relation/node names, never as a
+// structured `host` field, so redaction reaches those residuals too. Pseudonyms
+// (`host-1`) match neither pattern, keeping the scan idempotent.
 const HOST_PATTERNS = [
   // EC2-style ip-10-1-2-3 with an optional dotted domain (ip-10-1-2-3.ec2.internal).
   // Each domain label must start with an alphanumeric, so a trailing sentence
@@ -95,6 +94,31 @@ function redactFailureGroups   (node   )    {
   return node;
 }
 
+// A stage's failure reason is Spark's free-text message and can carry file paths and data values
+// too, so it is replaced outright, like a failure group's message: a `stageFailed` finding's
+// `valueText`, the `stageFailureReason` of a stage record, a job's `exception` (the run outcome's
+// fallback reason), and the report's `failureReason` copy. Walking by key name, like
+// collectHostFields, needs no path list. Returns a fresh tree.
+const REASON_KEYS = new Set(['stageFailureReason', 'exception', 'failureReason']);
+function redactStageFailureReasons   (node   )    {
+  if (Array.isArray(node)) return node.map((n) => redactStageFailureReasons(n))     ;
+  if (node && typeof node === 'object') {
+    const isStageFailed = (node                      ).type === 'stageFailed';
+    const out                          = {};
+    for (const [k, v] of Object.entries(node)) {
+      const isReason = REASON_KEYS.has(k) || (isStageFailed && k === 'valueText');
+      out[k] = isReason && typeof v === 'string' ? REDACTED_TEXT : redactStageFailureReasons(v);
+    }
+    return out     ;
+  }
+  return node;
+}
+
+// Both passes for free text no host/app-id pattern recognizes.
+function redactFailureText   (node   )    {
+  return redactStageFailureReasons(redactFailureGroups(node));
+}
+
 // Spark config keys ending in `host`/`hostname` (e.g. spark.driver.host,
 // spark.yarn.am.hostname) carry plain FQDN host names that neither
 // HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
@@ -122,7 +146,7 @@ function collectConfigHostValues(config                                    , hos
                             
                                                                           
                                                                               
-                                             
+                                                                   
                                                                                
                                                                               
                  
@@ -188,10 +212,19 @@ function applyReplacements   (node   , ids
   return deepReplace(node, merged)     ;
 }
 
+// The app name identifies a job as much as its id, so a redacted name becomes the id's
+// pseudonym, as list_runs does. Only the structured field is replaced, never substrings: a
+// short name ("t", "etl") would otherwise corrupt every string it happens to occur in.
+function withRedactedName                                                        (app   )    {
+  return app.name == null ? app : { ...app, name: app.id ?? null };
+}
+
 export function redactReport                            (input   )    {
-  const report = redactFailureGroups(input);
+  const report = redactFailureText(input);
   const { appIds, hosts } = collectIds(report);
-  return applyReplacements(report, { appIds, hosts });
+  const out = applyReplacements(report, { appIds, hosts });
+  const app = out.summary?.app;
+  return app ? { ...out, summary: { ...out.summary, app: withRedactedName(app) } } : out;
 }
 
 // Run-comparison counterpart: no single app-id *field* to pseudonymize
@@ -218,7 +251,7 @@ export function redactComparison   (comparison   )    {
                                                                                        
 
 function redactRunTree                   (input   )    {
-  const data = redactFailureGroups(input);
+  const data = redactFailureText(input);
   const appIds = new Set        ();
   const hosts = new Set        ();
   const appId = data.app?.id;
@@ -229,7 +262,12 @@ function redactRunTree                   (input   )    {
   collectHostFields(data.configFindings, hosts);
   collectConfigHostValues(data.app?.config, hosts);
   scanTokens(data, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
-  return applyReplacements(data, { appIds, hosts });
+  const out = applyReplacements(data, { appIds, hosts });
+  if (!out.app) return out;
+  const app = withRedactedName(out.app);
+  // spark.app.name repeats the name in the config the HTML export ships.
+  if (app.config?.['spark.app.name'] == null) return { ...out, app };
+  return { ...out, app: { ...app, config: { ...app.config, 'spark.app.name': app.id ?? '' } } };
 }
 
 /** A run's model and findings with every identifier pseudonymized. Redact this before anything derives
@@ -264,6 +302,9 @@ export function redactRunModel(
       executors: redacted.executors,
       runAggregates: redacted.runAggregates                             ,
       evidenceAvailability: redacted.evidenceAvailability                                    ,
+      // Counts and execution ids, nothing to pseudonymize.
+      skippedLines: appModel.skippedLines,
+      unreadableSqlExecutions: appModel.unreadableSqlExecutions,
     }            ,
     catalog: redacted.catalog,
     configFindings: redacted.configFindings,