sparkforensics-server 0.5.2 → 0.8.1

package/vendor-core/mcp-server-factory.js

@@ -2,10 +2,12 @@ import { z } from 'zod';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
                                                                          
 import {
-  resolveOrCreateRun, diagnoseRun, getRunSummary, compareRuns, getFindingEvidence, getFindingDocumentation, getReferenceDoc, evaluateBudgetsForRun,
+  resolveOrCreateRun, diagnoseRun, diagnoseHeadline, getRunSummary, compareRuns, getFindingEvidence, getFindingDocumentation, getReferenceDoc, evaluateBudgetsForRun,
+                         
 } from './mcp-tools.js';
 import { listRuns } from './list-runs.js';
                                                          
+                                                       
 import { MAX_NORMALIZE_PATTERN_LENGTH, MAX_NORMALIZE_PATTERNS } from './stage-alignment.js';
 
 const sourceSchema = z.union([
@@ -42,7 +44,7 @@ function toolErrorResult(error                                    )
   };
 }
 
-// diagnoseRun/compareRuns may carry a `markdown` field (only when the caller asked for
+// compareRuns may carry a `markdown` field (only when the caller asked for
 // format: 'md'); it never belongs in structuredContent (whose shape must stay stable regardless of
 // format), so it's stripped here and, when present, becomes content[0].text instead of JSON.stringify.
 function toolResultWithMarkdown                  (promise                                    )                          {
@@ -55,16 +57,40 @@ function toolResultWithMarkdown                  (promise
   );
 }
 
+// diagnose_run's own mapping: with `format: 'md'` the Markdown is the content and structuredContent
+// shrinks to a few headline fields, so a client that stores it does not store the whole report.
+function diagnoseToolResult(promise                            )                          {
+  return promise.then(
+    (value) => {
+      const { markdown, ...structuredContent } = value;
+      return markdown !== undefined
+        ? toCallToolResult(markdown, diagnoseHeadline(value))
+        : toCallToolResult(JSON.stringify(structuredContent), structuredContent                           );
+    },
+    toolErrorResult,
+  );
+}
+
 // Counterpart for the tools that never produce a markdown field: its own mapping, so there's no
 // need to cast T to pretend it might carry a `markdown` field it never does.
 function toolResult                  (promise            )                          {
   return promise.then((value) => toCallToolResult(JSON.stringify(value), value                           ), toolErrorResult);
 }
 
+// The handshake version of a server built without a `generator`: no package to read a version from.
+const UNKNOWN_VERSION = '0.0.0';
+
+// Shared by every tool that returns a run handle.
+const RUN_ID_NOTE = 'runId is a handle for the run this server process has cached. It differs per tool by design (compare_runs returns runIdA and runIdB, evaluate_budgets the runId of its first run) and per server process, so match runs by app.id and app.attemptId, not by runId.';
+
 /** `thresholds`: the bin's --thresholds overrides, applied to every analyzing tool for the life
- * of the server. Omitted, every tool runs the specification defaults. */
-export function createMcpServer({ thresholds }                                      = {})            {
-  const server = new McpServer({ name: 'sparkforensics', version: '1.0.0' });
+ * of the server. Omitted, every tool runs the specification defaults.
+ * `generator`: the host package's name, version and core build id. The server reports its version
+ * in the MCP handshake and every diagnose_run and compare_runs result carries the block. */
+export function createMcpServer(
+  { thresholds, generator }                                                                   = {},
+)            {
+  const server = new McpServer({ name: 'sparkforensics', version: generator?.version ?? UNKNOWN_VERSION });
 
   server.registerTool('list_runs', {
     description: 'List candidate Spark event-log runs from a local directory or a Spark History Server, before diagnosing one with the other tools.',
@@ -72,16 +98,22 @@ export function createMcpServer({ thresholds }
   }, (params) => toolResult(listRuns(params)));
 
   server.registerTool('diagnose_run', {
-    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, the checks the log lacked the data to run, and the writeTargets, metrics and effectiveConf blocks of the CLI report. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
+    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings, each with its lead finding), findings, an impact-ranked fix recommendation rollup, and the writeTargets, metrics and effectiveConf blocks of the CLI report. view "compact" (the default) lists one short row per finding and leaves out the clean-check and not-checked lists; view "full" returns every finding with its remediation text and evidence, and those lists. Every finding row has a key that stays the same for the same finding in a rerun of the job (compare_runs rows list the same keys), and an id for get_finding_evidence. With format "md" the content is Markdown and structuredContent holds a few headline fields. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated. The result names the Spark application (app) and the build that wrote it (generator). ' + RUN_ID_NOTE,
     inputSchema: {
       ...runRefSchema, redact: z.boolean().optional(),
+      view: z.enum(['compact', 'full']).optional()
+        .describe('compact (default): one short row per finding. full: the CLI report\'s rows, clean checks and not-checked checks.'),
       include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
       impactBand: z.array(z.string()).optional(), type: z.array(z.string()).optional(), stageId: z.number().int().optional(),
+      minBand: z.enum(['critical', 'warning', 'info']).optional()
+        .describe('List only findings at this band or a more severe one.'),
+      maxFindings: z.number().int().nonnegative().optional()
+        .describe('List at most this many findings, most severe first. findingsOmitted counts the ones left out; each verdict step still carries its lead finding.'),
       ...formatSchema,
     },
-  }, ({ source, runId, redact, include, format, impactBand, type, stageId }) => toolResultWithMarkdown(
+  }, ({ source, runId, redact, include, format, impactBand, type, stageId, view, minBand, maxFindings }) => diagnoseToolResult(
     resolveOrCreateRun({ source, runId }).then(({ runId: id }) =>
-      diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId, thresholds })),
+      diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId, thresholds, generator, view: view ?? 'compact', minBand, maxFindings })),
   ));
 
   server.registerTool('get_run_summary', {
@@ -92,7 +124,7 @@ export function createMcpServer({ thresholds }
   ));
 
   server.registerTool('compare_runs', {
-    description: 'Compare two runs: the verdict, metric deltas (metrics), categorized findings delta (findings) and the stage pairs (stagePairs) with the share of executor run time they cover (runtimeCoverage). confidence is ok, low or insufficient. metricDeltas and findingsDelta repeat metrics and findings under their earlier names and will be removed.',
+    description: 'Compare two runs: the verdict, metric deltas (metrics), categorized findings delta (findings), the stage pairs (stagePairs) with the share of executor run time they cover (runtimeCoverage), and comparability: the configuration differences, plan equality, data volume, executor capacity and the factors that bridge the wall-clock difference. confidence is ok, low or insufficient. metricDeltas and findingsDelta repeat metrics and findings under their earlier names and will be removed. The result names both Spark applications (baselineApp, candidateApp) and the build that wrote it (generator). Each findings row lists its stage names in stages and the same names without call-site directories in stageLabels. ' + RUN_ID_NOTE,
     inputSchema: {
       runIdA: z.string().optional(), sourceA: sourceSchema.optional(),
       ...secondRunRefSchema,
@@ -101,14 +133,18 @@ export function createMcpServer({ thresholds }
         .describe('Extra blocks to return. stagePairs: one row per paired stage with its deltas, which can be large.'),
       normalizePath: z.array(z.string().max(MAX_NORMALIZE_PATTERN_LENGTH)).max(MAX_NORMALIZE_PATTERNS).optional()
         .describe('Regular expressions whose every match in a plan node\'s text is replaced with a fixed token before stages are paired, so run-specific text (an output directory that differs per run) does not keep the same stage from pairing. A generic pattern, not only for paths. Affects stage pairing, runtimeCoverage and confidence, never findings. A pattern that backtracks catastrophically can stall the server: keep them simple and anchored.'),
+      ignoreConf: z.array(z.string().max(MAX_NORMALIZE_PATTERN_LENGTH)).max(MAX_NORMALIZE_PATTERNS).optional()
+        .describe('Regular expressions matched against Spark configuration keys. A key that matches is left out of comparability.conf, for keys that differ by design such as a per-run output path. Per-run identity keys (application id, start time, driver address) are always left out.'),
+      noiseBand: z.number().min(0).max(1).optional()
+        .describe('Share of the baseline value under which a metric row is marked withinNoise (default 0.02). Never changes a row\'s direction.'),
       ...formatSchema,
     },
-  }, ({ runIdA, sourceA, runIdB, sourceB, redact, include, normalizePath, format }) => toolResultWithMarkdown(
-    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds, normalizePath, include }),
+  }, ({ runIdA, sourceA, runIdB, sourceB, redact, include, normalizePath, ignoreConf, noiseBand, format }) => toolResultWithMarkdown(
+    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds, normalizePath, ignoreConf, noiseBand, include, generator }),
   ));
 
   server.registerTool('evaluate_budgets', {
-    description: 'Evaluate a run against pass/fail thresholds, optionally against a baseline run for regression budgets. With two runs, absolute budgets apply to the candidate (sourceB/runIdB), matching the CLI. A run with no ApplicationEnd always adds an inconclusive run-complete result.',
+    description: 'Evaluate a run against pass/fail thresholds, optionally against a baseline run for regression budgets. With two runs, absolute budgets apply to the candidate (sourceB/runIdB), matching the CLI. A run with no ApplicationEnd always adds an inconclusive run-complete result. ' + RUN_ID_NOTE,
     inputSchema: {
       source: sourceSchema.optional().describe('The run to evaluate. When sourceB/runIdB is given, this is the regression baseline instead, and the absolute budgets apply to sourceB/runIdB.'),
       runId: runRefSchema.runId.describe('Same as `source`, referencing an already-resolved run by id.'),

package/vendor-core/mcp-tools.js

@@ -1,17 +1,21 @@
 import { resolve as resolvePath, join, dirname } from 'node:path';
-import { existsSync, statSync, readFileSync } from 'node:fs';
+import { existsSync, statSync, readFileSync, readdirSync,            } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { randomUUID } from 'node:crypto';
 import { collectRun } from './cli/collect-run.js';
 import { deriveEvidenceAvailability } from './evidence-availability.js';
 import { resolveFromShs, DEFAULT_MAX_ARCHIVE_BYTES, DEFAULT_IDLE_TIMEOUT_MS } from './shs-load.js';
 import { mcpError } from './mcp-error.js';
-import { buildEvidenceReport, runFindings, toFindingsFilter,                                                                                                               } from './evidence-report.js';
+import {
+  buildEvidenceReport, runFindings, toFindingsFilter,
+                                                                                                                                                       
+} from './evidence-report.js';
 import { computeWallClock } from './wall-clock.js';
 import { renderComparisonMarkdown,                        } from './run-comparison.js';
 import { buildComparisonOutput,                       } from './comparison-output.js';
 import { compileNormalizePatterns } from './stage-alignment.js';
-import { runOutputBlocks } from './run-output.js';
+import { compileConfIgnorePatterns } from './comparability.js';
+import { runOutputBlocks,                      } from './run-output.js';
 import { evaluateBudgets,                                       } from './cli/budgets.js';
 import { assertStageBudgetMetric, combineRegressionBudgets } from './cli/regression-budgets.js';
 import { FINDING_NAMES, titleCase } from './finding-names.js';
@@ -20,7 +24,7 @@ import { DETECTORS,                         } from './detectors.js';
 import { tunedDetectors } from './threshold-overrides.js';
 import { tuningDocSlugForAnchor, pageForAnchor } from './docs-config.js';
 import { typeTag } from './format-utils.js';
-                                                                                   
+                                                                                               
                                                
 
                                                                                                      
@@ -53,7 +57,7 @@ import { typeTag } from './format-utils.js';
                                                                 
                  
                  
-                                                                               
+                                                                                 
                                               
                                             
  
@@ -72,18 +76,70 @@ const CACHE_TTL_MS = envInt('SPARKFORENSICS_MCP_CACHE_TTL_MS', 15 * 60 * 1000);
 
 const byRunId = new Map                    ();   // runId -> { appModel, cacheKey, lastAccess }
 const byCacheKey = new Map                (); // cacheKey -> runId
+// Built comparisons, keyed by both runs and every option that changes the output. Each entry follows
+// its runs' TTL and eviction (deleteRunEntry), and the map is capped on its own: a comparison holds
+// a row per paired stage, and 8 runs allow 56 ordered pairs.
+const COMPARISON_CACHE_CAP = 16;
+                                                                                                                  
+const comparisonCache = new Map                              ();
+// Threshold overrides are one fixed object per server process (or DEFAULT for none): identity keys them.
+const thresholdsIds = new WeakMap                ();
+let nextThresholdsId = 1;
+function thresholdsKey(thresholds                                )         {
+  if (!thresholds) return 0;
+  let id = thresholdsIds.get(thresholds);
+  if (id === undefined) {
+    id = nextThresholdsId++;
+    thresholdsIds.set(thresholds, id);
+  }
+  return id;
+}
+
 const pendingByCacheKey = new Map                                                        (); // cacheKey -> in-flight Promise<{runId, appModel}>
 
 export function pathCacheKey(path        )         {
   const resolved = resolvePath(path);
   if (!existsSync(resolved)) throw mcpError('invalid-event-log', `No such file: ${resolved}`);
+  const stat = statSync(resolved);
+  // A rolling event-log directory grows by appending to its live events_N file, which leaves the
+  // directory's own mtime and size alone: the key covers the files inside it.
+  if (stat.isDirectory()) return `path:${resolved}:${directoryStatKey(resolved)}`;
   // Size narrows a same-millisecond mtime collision; ctime narrows the case where a copy tool
   // (rsync --preserve-times, tar) restores an identical mtime+size for different content: ctime
   // can't be set by the copying tool, so it still reflects when the file landed on disk.
-  const { mtimeMs, ctimeMs, size } = statSync(resolved);
-  return `path:${resolved}:${mtimeMs}:${ctimeMs}:${size}`;
+  return `path:${resolved}:${fileStatKey(stat)}`;
+}
+
+function fileStatKey({ mtimeMs, ctimeMs, size }       )         {
+  return `${mtimeMs}:${ctimeMs}:${size}`;
 }
 
+// The file count, newest mtime and ctime, and total size of the files under a directory (the
+// parts of a rolling log, which Spark appends to in place): any part growing, arriving or going
+// changes it.
+function directoryStatKey(dir        )         {
+  let files = 0, mtimeMs = 0, ctimeMs = 0, size = 0;
+  const walk = (path        , depth        )       => {
+    for (const entry of readdirSync(path, { withFileTypes: true })) {
+      const child = join(path, entry.name);
+      if (entry.isDirectory()) {
+        if (depth < ROLLING_LOG_DEPTH) walk(child, depth + 1);
+        continue;
+      }
+      const stat = statSync(child);
+      files++;
+      mtimeMs = Math.max(mtimeMs, stat.mtimeMs);
+      ctimeMs = Math.max(ctimeMs, stat.ctimeMs);
+      size += stat.size;
+    }
+  };
+  walk(dir, 0);
+  return `dir:${files}:${mtimeMs}:${ctimeMs}:${size}`;
+}
+// A rolling log's parts sit in the directory the path names, or one level down in its
+// eventlog_v2_* subdirectory.
+const ROLLING_LOG_DEPTH = 1;
+
 function shsCacheKey({ shsBaseUrl, appId, attemptId }                                                           )         {
   return `shs:${shsBaseUrl}:${appId}:${attemptId ?? ''}`;
 }
@@ -99,6 +155,10 @@ function touch(runId        )       {
 
 function deleteRunEntry(id        , entry            )       {
   byRunId.delete(id);
+  // A comparison outlives neither of its runs: its entry goes with the first one evicted.
+  for (const [key, cached] of comparisonCache) {
+    if (cached.runIdA === id || cached.runIdB === id) comparisonCache.delete(key);
+  }
   if (byCacheKey.get(entry.cacheKey) === id) byCacheKey.delete(entry.cacheKey);
 }
 
@@ -121,6 +181,11 @@ function evictOverflow()       {
   }
 }
 
+/** How many built comparisons the cache holds: the occupancy the cap and eviction rules bound. */
+export function comparisonCacheSize()         {
+  return comparisonCache.size;
+}
+
 export function getCachedAppModel(runId        )           {
   evictStale();
   const entry = byRunId.get(runId);
@@ -166,6 +231,8 @@ export async function resolveOrCreateRun(
       const appModel = 'path' in source
         ? await resolveFromPath(source.path)
         : (await resolveFromShs(source.shsBaseUrl, source.appId, source.attemptId, { fetchImpl, maxArchiveBytes, idleTimeoutMs })).appModel;
+      // The attempt the caller asked the History Server for, when the log itself does not name one.
+      if (!('path' in source) && source.attemptId && appModel.app && !appModel.app.attemptId) appModel.app.attemptId = source.attemptId;
 
       const newRunId = randomUUID();
       byRunId.set(newRunId, { appModel, cacheKey, lastAccess: Date.now() });
@@ -180,30 +247,83 @@ export async function resolveOrCreateRun(
   return resolution;
 }
 
+/** `compact`: an index, one short row per finding and no clean or not-run check lists. `full`: the
+ * report as the CLI writes it. */
+                                              
+
+/** A finding as `view: 'compact'` lists it: enough to rank it, name it and pair it across reruns.
+ * The rest of the row (recommendation, evidence, remediation, estimate) comes from
+ * `get_finding_evidence` by `id`. */
+                                    
+                                                                          
+                                                                                    
+                                                                  
+                                                 
+                                    
+ 
+
+export function compactFindingRow(row            )                    {
+  return {
+    id: row.id, key: row.key, type: row.type, tag: row.tag, name: row.name,
+    impactBand: row.impactBand, stageId: row.stageId, actionLabel: row.actionLabel,
+    metric: row.metric ?? null, value: row.value ?? null,
+    ...(row.valueText !== undefined ? { valueText: row.valueText } : {}),
+    ...(row.impact !== undefined ? { impact: row.impact, impactMeaning: row.impactMeaning ?? null } : {}),
+    // Kept: a tuned finding's impact figure is uncalibrated, and a reader must see that on its row.
+    ...(row.tunedThresholds !== undefined ? { tunedThresholds: row.tunedThresholds } : {}),
+  };
+}
+
+/** A verdict step with its lead finding's row, which `findings` may not list under `maxFindings` or
+ * `minBand`. Null for a step whose lead has no id. */
+                                                                                                          
+
+                                 
+                                    
+                                                                         
+                                                  
+                                                                    
+                          
+                       
+                                       
+                                                    
+                           
+                                                                     
+                                                                                 
+                                                                                                                
+
 // `thresholds` on every analyzing tool below: the server's --thresholds overrides, fixed for the
 // process by createMcpServer(). A client can't set them per call; the dashboard never has them.
+// `view` defaults to `full` here; the MCP tool defaults it to `compact`.
 export function diagnoseRun(runId        , opts    
                                                                                                           
-                                                                                            
- )   
-                                                                                                      
-                                                                                                         
-                                                    
-                                                                                 
-                                                                                                                {
+                                                                                                                         
+                                                                  
+ )                    {
   const appModel = getCachedAppModel(runId);
-  const findingsFilter = toFindingsFilter(opts?.impactBand, opts?.type, opts?.stageId);
-  const { json, markdown } = buildEvidenceReport(appModel, {
+  const view = opts?.view ?? 'full';
+  const findingsFilter = toFindingsFilter(opts?.impactBand, opts?.type, opts?.stageId, opts?.minBand, opts?.maxFindings);
+  const { json, markdown, findingsMatched } = buildEvidenceReport(appModel, {
     redact: opts?.redact, markdown: opts?.markdown, findingsFilter, thresholds: opts?.thresholds,
   });
+  // The lead rows come from the unfiltered report (cached, so this costs no second analysis).
+  const allRows = findingsFilter
+    ? buildEvidenceReport(appModel, { redact: opts?.redact, markdown: false, thresholds: opts?.thresholds }).json.findings
+    : json.findings;
+  const shape = (row            )                                 => (view === 'compact' ? compactFindingRow(row) : row);
+  const steps = json.verdict.steps.map((step)                      => {
+    const lead = step.leadFindingId == null ? undefined : allRows.find((row) => row.id === step.leadFindingId && row.type === step.type);
+    return { ...step, leadFinding: lead ? shape(lead) : null };
+  });
   const include = opts?.include ?? [];
   const tuned = json.summary.tunedThresholds;
   return {
-    runId, verdict: json.verdict, findings: json.findings, recommendations: json.recommendations, cleanChecks: json.cleanChecks,
-    notRunChecks: json.notRunChecks,
+    runId, view, verdict: { ...json.verdict, steps }, findings: json.findings.map(shape), findingsOmitted: findingsMatched - json.findings.length,
+    recommendations: json.recommendations,
+    ...(view === 'full' ? { cleanChecks: json.cleanChecks, notRunChecks: json.notRunChecks } : {}),
     // The CLI report's own blocks, so a run reads the same through either surface.
     writeTargets: json.writeTargets,
-    ...runOutputBlocks(appModel, { redact: opts?.redact, thresholds: opts?.thresholds }),
+    ...runOutputBlocks(appModel, { redact: opts?.redact, thresholds: opts?.thresholds, generator: opts?.generator }),
     runComplete: appModel.app?.endTime != null,
     // Top level too, so a client that never asks for `summary` still sees the run was tuned.
     ...(tuned ? { tunedThresholds: tuned } : {}),
@@ -214,6 +334,16 @@ export function diagnoseRun(runId        , opts
   };
 }
 
+/** The few fields `format: 'md'` returns as `structuredContent`: the Markdown carries the rest. */
+export function diagnoseHeadline(result                   )   
+                                                                                                                                 
+  {
+  return {
+    runId: result.runId, runComplete: result.runComplete, title: result.verdict.title,
+    findingCount: result.findings.length, findingsOmitted: result.findingsOmitted, recommendationCount: result.recommendations.length,
+  };
+}
+
 const DOCS_CONTENT_DIR = join(dirname(fileURLToPath(import.meta.url)), 'docs-content');
 
 // Both vendored corpora lead with their own heading line; strips the leading '#'/'###' markers, an
@@ -351,6 +481,7 @@ async function resolveAndAnalyze(
 
 export async function compareRuns(
   a        , b        , opts                                                                                                              
+                                                                                    
                                                                                                                   ,
 )                                                                                                           {
   // Before any log is read: a bad pattern refuses the call, as the CLI's usage error does.
@@ -359,6 +490,11 @@ export async function compareRuns(
   } catch (e) {
     throw mcpError('access-or-upstream-failure', `normalizePath: ${(e         ).message}`);
   }
+  try {
+    compileConfIgnorePatterns(opts?.ignoreConf);
+  } catch (e) {
+    throw mcpError('access-or-upstream-failure', `ignoreConf: ${(e         ).message}`);
+  }
   const [
     { runId: runIdA, appModel: appModelA, catalog: catalogA },
     { runId: runIdB, appModel: appModelB, catalog: catalogB },
@@ -368,11 +504,30 @@ export async function compareRuns(
   // interactive stage-detail drill-down, which none of compare/matchStages/metricDeltas/findingsDelta
   // read. Findings/metrics come from `catalog` and appModel.stages/sql, both fully populated. Produces
   // identical output to the dashboard; the MCP tool just never exposes per-task drill-down.
-  const { comparison, output } = buildComparisonOutput(
-    { label: runIdA, appModel: appModelA, catalog: catalogA },
-    { label: runIdB, appModel: appModelB, catalog: catalogB },
-    { redact: opts?.redact, normalizePath: opts?.normalizePath, view: opts?.include?.includes('stagePairs') ? 'full' : 'summary' },
-  );
+  const view = opts?.include?.includes('stagePairs') ? 'full' : 'summary';
+  const cacheKey = JSON.stringify([runIdA, runIdB, thresholdsKey(opts?.thresholds), !!opts?.redact, view, opts?.normalizePath ?? [], opts?.ignoreConf ?? [], opts?.noiseBand ?? null, opts?.generator ?? null]);
+  let built = comparisonCache.get(cacheKey)?.built;
+  if (built) {
+    // Re-insert so the oldest entry is the least recently used.
+    comparisonCache.delete(cacheKey);
+    comparisonCache.set(cacheKey, { runIdA, runIdB, built });
+  } else {
+    built = buildComparisonOutput(
+      { label: runIdA, appModel: appModelA, catalog: catalogA },
+      { label: runIdB, appModel: appModelB, catalog: catalogB },
+      { redact: opts?.redact, normalizePath: opts?.normalizePath, ignoreConf: opts?.ignoreConf, noiseBand: opts?.noiseBand, view, generator: opts?.generator },
+    );
+    // Either run can be evicted while the other one is still being read or parsed (the awaits
+    // above): an entry for a run that is gone would outlive its eviction, so it is only kept
+    // while both are cached.
+    if (byRunId.has(runIdA) && byRunId.has(runIdB)) {
+      comparisonCache.set(cacheKey, { runIdA, runIdB, built });
+      while (comparisonCache.size > COMPARISON_CACHE_CAP) comparisonCache.delete(comparisonCache.keys().next().value );
+    }
+  }
+  const { comparison } = built;
+  // A copy, so a caller that changes the result cannot change what the next caller is served.
+  const output = structuredClone(built.output);
 
   return {
     runIdA,

package/vendor-core/median.js

@@ -1,5 +1,5 @@
-// The textbook median of values already sorted ascending: the middle value for an odd count, the
-// mean of the two middle values for an even one. 0 for an empty list.
+// Median of values already sorted ascending: the mean of the two middle values on an even count.
+// 0 for an empty list.
 export function medianOfSorted(sorted                   )         {
   const n = sorted.length;
   if (n === 0) return 0;

package/vendor-core/nested-loop-join.js

@@ -0,0 +1,73 @@
+// Reads a nested-loop join's operator, join type and condition from its plan line, and the row
+// counts around it from the executors' "number of output rows" metric.
+//
+// Spark plans a join with no equi-join keys as a BroadcastNestedLoopJoin when one side can be
+// broadcast, else a CartesianProduct for an inner-like join (JoinSelection in
+// sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala, v3.5.0). Both
+// compare every left row with every right row.
+                                           
+
+                                                                                
+
+                                      
+                               
+                                                                                                
+                          
+                                                                                     
+                           
+ 
+
+const OUTPUT_ROWS = 'number of output rows';
+
+// BroadcastNestedLoopJoin <BuildLeft|BuildRight>, <JoinType>[, <condition>]
+const BNLJ_LINE = /^BroadcastNestedLoopJoin\s+Build(?:Left|Right),\s*(\w+)(?:,\s*([\s\S]+))?$/;
+// CartesianProduct[ <condition>]: the operator only plans inner-like joins and prints no type.
+const CARTESIAN_LINE = /^CartesianProduct(?:\s+([\s\S]+))?$/;
+
+function cleanCondition(raw                    )                {
+  const text = (raw ?? '').replace(/#\d+L?/g, '').trim();
+  return text.length > 0 ? text : null;
+}
+
+/** The nested-loop operator, join type and condition behind a plan node, or null for any other node. */
+export function parseNestedLoopJoin(node                                   )                             {
+  const detail = (node.detail ?? '').trim();
+  if (node.name === 'BroadcastNestedLoopJoin') {
+    const m = detail.match(BNLJ_LINE);
+    return { operator: 'BroadcastNestedLoopJoin', joinType: m?.[1] ?? null, condition: cleanCondition(m?.[2]) };
+  }
+  if (node.name === 'CartesianProduct') {
+    const m = detail.match(CARTESIAN_LINE);
+    return { operator: 'CartesianProduct', joinType: 'Inner', condition: cleanCondition(m?.[1]) };
+  }
+  return null;
+}
+
+// Operators that pass every row of their one child through, so the child's row count is theirs.
+// A node outside this list that reports no row metric (Expand, Union, Limit...) can change the
+// count, so the walk stops there rather than guess.
+const ROW_PRESERVING = new Set([
+  'WholeStageCodegen', 'InputAdapter', 'Project', 'Sort', 'Exchange', 'BroadcastExchange',
+  'ShuffleQueryStage', 'BroadcastQueryStage', 'AQEShuffleRead', 'CustomShuffleReader',
+  'ColumnarToRow', 'AdaptiveSparkPlan',
+]);
+
+/** The rows a subtree produced: the nearest "number of output rows" at or below `node`, walking
+ * through row-preserving single-child operators. Null when the log reports none for it. */
+export function outputRowsOf(node          )                {
+  let current                       = node;
+  while (current) {
+    const metric = current.metrics?.find((m) => m.name === OUTPUT_ROWS);
+    if (metric) return metric.value;
+    const preserving = ROW_PRESERVING.has(current.name) || /^WholeStageCodegen/.test(current.name);
+    if (!preserving || current.children.length !== 1) return null;
+    current = current.children[0];
+  }
+  return null;
+}
+
+/** The node's own "number of output rows" value, null when absent. */
+export function ownOutputRows(node          )                {
+  const metric = node.metrics?.find((m) => m.name === OUTPUT_ROWS);
+  return metric ? metric.value : null;
+}

package/vendor-core/occupancy.js

@@ -193,8 +193,7 @@ export function stragglerFixLongestTaskMs(stage           )         {
                                                                                           
                                 
                                                                                                   
-                                                                                                  
-                                                                                  
+                                                         
                              
                                                                                                 
                                                                                   

package/vendor-core/parser-worker.js

@@ -251,7 +251,7 @@ export async function runParseFiles(
   emitParseCompletion(state, emit, linesProcessed);
 }
 
-// ─── Worker message bus (only active when running as a Web Worker) ──────────────
+// Message bus, active only when running as a Web Worker.
 
 const isWorker = typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope;
 

package/vendor-core/plan-graph-model.js

@@ -1,11 +1,11 @@
 // Flattens a resolved planTree into a { nodes, edges } graph shape for the
 // React Flow + Dagre plan graph view ("Plan graph view" in
 // docs-site/contributor-guide/architecture/drill-down/plan-graph.md). The Exchange
-// write/read split is done upstream now, in resolvePlanTree
-// (event-handlers.ts): this file just walks the already-split tree and maps
-// each real PlanNode onto a PlanGraphNodeData. sourceNodeId still lets a
-// write half point back to its read half's id, so aggregate logic over
-// nodes can group a pair without special-casing exchangeRole everywhere.
+// write/read split happens upstream in resolvePlanTree (event-handlers.ts):
+// this file walks the already-split tree and maps each PlanNode onto a
+// PlanGraphNodeData. sourceNodeId lets a write half point back to its read
+// half's id, so aggregate logic can group a pair without special-casing
+// exchangeRole.
 import { walkPlanTree } from './plan-tree-walk.js';
 import {
   classifyNode,
@@ -82,15 +82,9 @@ export function buildPlanGraphModel(
 
   const durationMap =
     sqlExec && sqlExecutionId != null ? buildDurationMap(planTree, appModel, sqlExec, sqlExecutionId, durationMode) : null;
-  // A second, always-exclusive map, independent of the requested `durationMode`.
-  // Exclusive shares are a true, non-overlapping partition of each paired
-  // stage's wall time (computeExclusiveSharesForPairs), so their sum is a
-  // stable normalization total. PlanGraphCanvas needs this alongside
-  // `durationMap` (the mode-selected values actually shown) because summing
-  // `durationMap` itself would double/triple-count under 'inclusive' mode,
-  // where an ancestor's share already folds in every descendant's share.
-  // durationMode === 'exclusive' already computed the same map above; only
-  // redo the work when the requested mode actually differs.
+  // Always-exclusive map, independent of `durationMode`: the stable total a
+  // percentage denominator sums (see PlanGraphNodeData.exclusiveDurationShare).
+  // Reuses `durationMap` when the requested mode is already exclusive.
   const exclusiveDurationMap =
     durationMode === 'exclusive'
       ? durationMap
@@ -110,14 +104,12 @@ export function buildPlanGraphModel(
     }
   }
 
-  // resolvePlanTree (event-handlers.ts) now prefixes every id with its owning
-  // execution, but this filter stays as defense in depth: a finding is scoped
-  // to the execution whose graph is being built before it's indexed by node
-  // id, so a stale/synthetic finding whose planNodeIds happen to collide with
-  // this tree's ids (e.g. hand-built test fixtures, or data from before the
-  // prefix existed) still can't badge onto the wrong tree. No linked SQL
-  // execution means no findings attach, matching the sqlExec-falsy behavior
-  // used elsewhere in this function (durationMap, linkedStageIds).
+  // resolvePlanTree (event-handlers.ts) prefixes every id with its owning
+  // execution, but a finding is also scoped to the execution whose graph is
+  // being built, as defense in depth: a synthetic finding whose planNodeIds
+  // collide with this tree's ids (e.g. hand-built test fixtures) cannot badge
+  // onto the wrong tree. No linked SQL execution means no findings attach,
+  // like durationMap and linkedStageIds above.
   const findingsByNodeId = new Map                   ();
   if (sqlExecutionId != null) {
     for (const finding of findings) {

package/vendor-core/plan-node-detail.js

@@ -164,9 +164,8 @@ export function buildDurationMap(
     const s = appModel.stages.get(id);
     if (s) stagesById.set(id, { submittedAt: s.submittedAt, completedAt: s.completedAt });
   }
-  // attributeStageDurationToPlan reads its 3rd arg's own `.stageIds` field;
-  // override it with the derived array rather than the (always-empty on real
-  // data) one already on `sqlExec`.
+  // attributeStageDurationToPlan reads `.stageIds`; use the derived array, not sqlExec's own
+  // (see contributor-guide/architecture/detector-contract/plan-attribution.md).
   return durationMode === 'inclusive'
     ? attributeStageDurationToPlanInclusive(planTree, stagesById, { ...sqlExec, stageIds })
     : attributeStageDurationToPlan(planTree, stagesById, { ...sqlExec, stageIds });