sparkforensics-server 0.4.0 → 0.5.1

package/vendor-core/run-output.js

@@ -0,0 +1,26 @@
+import { runFindings } from './evidence-report.js';
+import { redactRunModel } from './redact.js';
+import { computeRunMetrics,                 } from './run-metrics.js';
+import { buildEffectiveConf,                    } from './effective-conf.js';
+                                                         
+                                           
+
+/** The CLI report's `metrics` and `effectiveConf` blocks, shared with MCP diagnose_run. Under
+ * `redact` they come from the run redacted with the report's own inputs, so host and app
+ * pseudonyms match the report's. `confKeys`/`confRedactRegex` narrow and mask `effectiveConf`. */
+export function runOutputBlocks(
+  appModel          ,
+  { redact, thresholds, confKeys, confRedactRegex }   
+                                                                                                     
+    = {},
+)                                                               {
+  let blocksModel = appModel;
+  if (redact) {
+    const { catalog, config } = runFindings(appModel, thresholds);
+    blocksModel = redactRunModel(appModel, catalog, config).appModel;
+  }
+  return {
+    metrics: computeRunMetrics(blocksModel, thresholds),
+    effectiveConf: buildEffectiveConf(blocksModel.app, { keys: confKeys, userPattern: confRedactRegex }),
+  };
+}

package/vendor-core/run-totals.js

@@ -0,0 +1,24 @@
+// The run-level sums every surface reads: the CLI metrics block and the run comparison (so the
+// dashboard's comparison view and the MCP compare_runs tool) share these, so one name never means
+// two figures.
+import { nsToMs } from './format-utils.js';
+                                        
+
+/** Each stage followed by the work its own figures leave out, shaped like a stage: the attempts a
+ * resubmit replaced, and task attempts the stage record dropped (a failed attempt's late tasks,
+ * failed retries, losing speculative copies). Summing the result counts every attempt's work,
+ * failed ones too. A stage's own record, which the detectors and per-stage views read, is left as
+ * it is. */
+export function withEarlierAttempts(stages         )          {
+  return stages.flatMap((s) => [s, ...[s.earlierAttempts, s.lateAttemptWork]
+    .filter((work) => work != null)
+    .map(({ durationMs, ...totals }) => ({ id: s.id, ...totals, submittedAt: 0, completedAt: durationMs ?? undefined }))]);
+}
+
+/** Summed executor CPU time in ms. Spark records it in nanoseconds and the parser reads an absent
+ * metric as 0, so stages that all read 0 never recorded it (older Spark): null, not 0. */
+export function totalExecutorCpuMs(stages                                          )                {
+  let ns = 0;
+  for (const s of stages) if (typeof s.executorCpuTime === 'number' && Number.isFinite(s.executorCpuTime) && s.executorCpuTime > 0) ns += s.executorCpuTime;
+  return ns > 0 ? nsToMs(ns) : null;
+}

package/vendor-core/run-verdict.js

@@ -236,13 +236,12 @@ export function verdictSummary(eligible           , steps            , facts
   } else {
     sentences.push(`${plural(eligible.length, 'finding')} in ${plural(steps.length, 'place')}.`);
     const wallClock = steps[0].lead.impactEstimate?.wallClock;
-    if (isIdleCapacityStep(steps[0])) {
-      sentences.push('A smaller cluster or dynamic allocation would free the idle cores for other jobs.');
-    } else if (wallClock && facts.runMs != null) {
+    // An idle-capacity lead needs no sentence here: the title gives the idle share and step 1 the fix.
+    if (!isIdleCapacityStep(steps[0]) && wallClock && facts.runMs != null) {
       sentences.push(`The first fix could save up to ${formatDuration(wallClock.high)} of this ${formatDuration(facts.runMs)} run.`);
     }
     if (steps.some((step) => step.related.length > 0)) {
-      sentences.push('Findings in the same stage usually share one cause, so they are grouped together and their savings overlap rather than add up.');
+      sentences.push('Findings on one stage are grouped, and their savings overlap.');
     }
   }
   if (facts.incomplete) {

package/vendor-core/scorecard-estimates.js

@@ -45,6 +45,7 @@ export function getScorecardEstimates(appModel          )
     app: appModel.app,
     stages: appModel.stages,
     executorsAdded: appModel.executors.added,
+    executorsRemoved: appModel.executors.removed,
     runAggregates: appModel.runAggregates,
   });
 

package/vendor-core/session-snapshot.js

@@ -24,6 +24,8 @@ import { isSupportedEvidenceAvailability } from './evidence-availability.js';
                          
                                       
                                                     
+                        
+                                     
                      
                                  
  
@@ -44,6 +46,8 @@ export function captureSnapshot(
     jobs: new Map(appModel.jobs),
     runAggregates: appModel.runAggregates,
     evidenceAvailability: appModel.evidenceAvailability,
+    skippedLines: appModel.skippedLines,
+    unreadableSqlExecutions: appModel.unreadableSqlExecutions && [...appModel.unreadableSqlExecutions],
     catalog: [...catalog],
     taskData: new Map(taskDataCache),
   };
@@ -71,6 +75,9 @@ export function applySnapshot(
   appModel.evidenceAvailability = isSupportedEvidenceAvailability(snapshot.evidenceAvailability)
     ? snapshot.evidenceAvailability
     : null;
+  appModel.skippedLines = snapshot.skippedLines;
+  if (snapshot.unreadableSqlExecutions) appModel.unreadableSqlExecutions = [...snapshot.unreadableSqlExecutions];
+  else delete appModel.unreadableSqlExecutions;
 
   taskDataCache.clear();
   for (const [k, v] of snapshot.taskData) taskDataCache.set(k, v);

package/vendor-core/shs-schemas.js

@@ -1,8 +1,8 @@
 import { z } from 'zod';
 
 // Proxy-level error envelope: the JSON body the local server's SHS proxy
-// (./shs-request.js) sends back on a non-OK upstream response, e.g.
-// `{ code: 'shs-unreachable' }`. This is NOT a SparkListener* event shape, so
+// (./proxy.js) sends back on a non-OK upstream response, e.g.
+// `{ code: 'upstream-unreachable' }`. This is NOT a SparkListener* event shape, so
 // it lives here rather than in event-schemas.ts. `.passthrough()` since the
 // proxy may attach extra debugging fields the consumer doesn't care about;
 // only `code` is read.

package/vendor-core/spark-memory.js

@@ -0,0 +1,17 @@
+// Parse a Spark memory-size string to MiB. Spark's JVM-memory configs use bytesConf(ByteUnit.MiB),
+// so a bare number means MiB. A k/m/g/t suffix sets the unit (trailing "b" redundant); a lone "b"
+// ("10b") means bytes.
+export function parseSparkMemoryMB(value         )                {
+  if (value == null) return null;
+  const m = String(value).trim().toLowerCase().match(/^([\d.]+)\s*([kmgt]?)(b?)$/);
+  if (!m) return null;
+  const n = parseFloat(m[1]);
+  if (!Number.isFinite(n)) return null;
+  switch (m[2]) {
+    case 'k': return Math.round(n / 1024);
+    case 'g': return Math.round(n * 1024);
+    case 't': return Math.round(n * 1024 * 1024);
+    case 'm': return Math.round(n);
+    default: return m[3] === 'b' ? Math.round(n / (1024 * 1024)) : Math.round(n);
+  }
+}

package/vendor-core/stage-alignment.js

@@ -0,0 +1,262 @@
+// The stage aligner: two snapshots and options in, typed stage pairs out. Pure (no I/O, no
+// dependency on the CLI, MCP or UI) and deterministic; swapping baseline and candidate mirrors the
+// pairs. `compareRuns` calls it, and every surface that compares runs reads its result
+// (`stagePairs`) instead of comparing stage keys.
+//
+// Pairing keys on `comparisonIdentity`: the `stageIdentity` recipe run with the comparison
+// normalizer, so the volatile tokens that make two runs of one job differ (staging directories,
+// dates, IN-lists, Delta log file counts) and the caller's `--normalize-path`
+// patterns do not split a stage in two. `stageIdentity` itself stays the frozen exact key.
+
+import { normalizeDetail } from './detectors.js';
+import { isDeltaLogRead, scanRelationId } from './plan-summary.js';
+import { withEarlierAttempts, totalExecutorCpuMs } from './run-totals.js';
+import { planNodesOfStage } from './stage-plan-nodes.js';
+import { identityIndexWith, pairEqualCounts, stageIdentityWith,                       } from './stage-identity.js';
+                                                             
+                                                  
+
+/** Version of the comparison block (`stagePairs`, `unmatched`, `replanned`, `bookkeepingStageIds`,
+ * `runtimeCoverage`). Additive changes keep it; a consumer refuses a version it does not know. */
+export const COMPARISON_SCHEMA_VERSION = 1;
+
+// ---- Comparison normalizer ------------------------------------------------------------------
+
+// Targeted patches for the volatile tokens that differ between two runs of one job. Each applies
+// on top of `normalizeDetail`, in this order. No blanket numeric or path stripping: those fuse
+// distinct stages of one run.
+export const COMPARISON_PATCHES                                                                = [
+  // Delta log and file-index file counts track how many files or commits exist, not the query:
+  // DeltaLogFileIndex(4 paths) vs DeltaLogFileIndex(8 paths).
+  { name: 'fileCounts', apply: (s) => s.replace(/FileIndex\(\d+ paths\)/g, 'FileIndex(n paths)') },
+  // Date and timestamp literals: the run's processing window.
+  { name: 'dates', apply: (s) => s.replace(/\d{4}-\d{2}-\d{2}(?:[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?)?/g, 'DATE') },
+  // IN-lists grow with the window. Spark prints small lists as `IN (a, b)` and large ones
+  // (InSet) as `INSET a, b` with no opening parenthesis.
+  {
+    name: 'inLists',
+    apply: (s) => s.replace(/\bIN \([^()]*\)/g, 'IN (...)').replace(/\bINSET [^()]*?(?=\))/g, 'INSET ...'),
+  },
+  // Random per-run staging directory segments (`/ds/aB3xYz/`): a 6-8 character alphanumeric path
+  // segment that mixes cases, or letters with digits. An ALL-CAPS or all-lowercase name survives,
+  // but a real directory that looks like that (`/output01/`, `/Region3/`) is rewritten too.
+  {
+    name: 'stagingDirs',
+    apply: (s) => s.replace(/\/([A-Za-z0-9]{6,8})(?=\/)/g, (m, seg        ) =>
+      (/[A-Z]/.test(seg) && /[a-z]/.test(seg)) || (/[A-Za-z]/.test(seg) && /\d/.test(seg)) ? '/TMP' : m),
+  },
+];
+
+/** What every match of a caller-supplied pattern is replaced with. */
+export const NORMALIZE_PATH_TOKEN = 'NORMALIZED';
+/** Longest caller pattern accepted. A pattern from an agent can backtrack catastrophically, so
+ * the length is capped; the cap does not make a bad pattern safe. */
+export const MAX_NORMALIZE_PATTERN_LENGTH = 200;
+export const MAX_NORMALIZE_PATTERNS = 16;
+
+/** Compiles caller-supplied normalize patterns (global, every match replaced). Throws an Error
+ * naming the offending pattern when one is too long, not a valid regular expression, or matches
+ * the empty string (which would insert the token between every character). */
+export function compileNormalizePatterns(patterns                               )           {
+  if (!patterns || patterns.length === 0) return [];
+  if (patterns.length > MAX_NORMALIZE_PATTERNS) {
+    throw new Error(`At most ${MAX_NORMALIZE_PATTERNS} normalize patterns are accepted, got ${patterns.length}.`);
+  }
+  return patterns.map((source) => {
+    if (typeof source !== 'string' || source.length === 0) throw new Error('A normalize pattern must be a non-empty string.');
+    if (source.length > MAX_NORMALIZE_PATTERN_LENGTH) {
+      throw new Error(`Normalize pattern "${source.slice(0, 40)}..." is ${source.length} characters; the limit is ${MAX_NORMALIZE_PATTERN_LENGTH}.`);
+    }
+    let re        ;
+    try {
+      re = new RegExp(source, 'g');
+    } catch (e) {
+      throw new Error(`Normalize pattern "${source}" is not a valid regular expression: ${(e         ).message}`);
+    }
+    if (re.test('')) throw new Error(`Normalize pattern "${source}" matches the empty string.`);
+    re.lastIndex = 0;
+    return re;
+  });
+}
+
+/** The detail normalizer the comparison keys use: caller patterns first, on the text as Spark
+ * printed it, then `normalizeDetail`, then the targeted patches. */
+export function comparisonDetailNormalizer(callerPatterns                    = [])                   {
+  return (detail) => {
+    let s = detail;
+    for (const re of callerPatterns) s = s.replace(re, NORMALIZE_PATH_TOKEN);
+    s = normalizeDetail(s);
+    for (const patch of COMPARISON_PATCHES) s = patch.apply(s);
+    return s;
+  };
+}
+
+// ---- Identity and bookkeeping ---------------------------------------------------------------
+
+/** `stageIdentity`'s recipe (normalized stage name plus the attributed nodes' fingerprint) with the
+ * comparison normalizer. Internal to the aligner: consumers key on a pair's `pairId`. */
+export function comparisonIdentity(
+  stage       , snapshot                              , normalizer                  , nodes             ,
+)         {
+  return stageIdentityWith(stage, snapshot, normalizer, nodes);
+}
+
+function isScanNode(node          )          {
+  return /^(Scan|BatchScan)\b/.test(node.name ?? '') || /FileScan|JDBCRelation/.test(node.detail ?? '');
+}
+
+/** True for a stage whose scans only read Delta log files or checkpoints: it has at least one scan
+ * and every scan is internal Delta metadata. Such a stage is table bookkeeping, not query work.
+ * A stage with no attributed plan nodes has no scans to test, so it is never flagged. */
+export function isDeltaBookkeepingStage(
+  stage                                      , sql                        , nodes             = planNodesOfStage(stage, sql),
+)          {
+  let scans = 0;
+  for (const node of nodes) {
+    if (!isScanNode(node)) continue;
+    const name = node.name ?? '', detail = node.detail ?? '';
+    if (scanRelationId(name, detail) !== null || !isDeltaLogRead(name, detail)) return false;
+    scans++;
+  }
+  return scans > 0;
+}
+
+// ---- Alignment ------------------------------------------------------------------------------
+
+export const PAIR_DELTA_METRICS = [
+  'executorRunTime', 'executorCpuTime', 'memoryBytesSpilled', 'diskBytesSpilled',
+  'inputBytes', 'outputBytes', 'shuffleReadBytes', 'shuffleWriteBytes',
+]         ;
+                                                                
+
+                                                                                                       
+
+                            
+                                                              
+                 
+                         
+                         
+                                              
+                                  
+                
+                                              
+ 
+
+                                 
+                     
+                                                                       
+                                                                
+                                                                                  
+                     
+                                                                                              
+                                                                          
+                                                                                                
+                                     
+                                 
+ 
+
+                               
+                                                               
+                                    
+ 
+
+// Sum of a field over the stage's own record and the work its figures leave out (earlier
+// attempts, a failed attempt's late tasks), the same all-attempt sums the run totals use.
+// null when no row carried a finite value, so an unrecorded field is not a false 0.
+function attemptSum(rows         , field                                             )                {
+  let sum = 0, present = false;
+  for (const row of rows) {
+    const v = row[field];
+    if (typeof v === 'number' && Number.isFinite(v)) { sum += v; present = true; }
+  }
+  return present ? sum : null;
+}
+
+function stageFigures(stage       )                                         {
+  const rows = withEarlierAttempts([stage]);
+  return {
+    executorRunTime: attemptSum(rows, 'executorRunTime'),
+    executorCpuTime: totalExecutorCpuMs(rows),
+    memoryBytesSpilled: attemptSum(rows, 'memoryBytesSpilled'),
+    diskBytesSpilled: attemptSum(rows, 'diskBytesSpilled'),
+    inputBytes: attemptSum(rows, 'inputBytes'),
+    outputBytes: attemptSum(rows, 'outputBytes'),
+    shuffleReadBytes: attemptSum(rows, 'shuffleReadBytes'),
+    shuffleWriteBytes: attemptSum(rows, 'shuffleWriteBytes'),
+  };
+}
+
+const ascending = (a        , b        )         => a - b;
+
+/** Pairs the stages of two runs. At this stage the pairing is `matchStages`' algorithm over
+ * `comparisonIdentity`: stages sharing an identity pair off by sorted id when each run has the same
+ * count of them, and every pair is `exact`. */
+export function alignStages(baseSnap                 , candSnap                 , options               = {})                 {
+  const normalizer = comparisonDetailNormalizer(options.normalizePath);
+  // One walk of the plan per stage serves the bookkeeping test and the identity.
+  const index = (snap                 ) => {
+    const nodesOf = new Map                    ();
+    const bookkeeping = new Set        ();
+    for (const [id, stage] of snap.stages) {
+      const nodes = planNodesOfStage(stage, snap.sql);
+      nodesOf.set(id, nodes);
+      if (isDeltaBookkeepingStage(stage, snap.sql, nodes)) bookkeeping.add(id);
+    }
+    const identities = identityIndexWith(
+      snap, (stage, id) => comparisonIdentity(stage, snap, normalizer, nodesOf.get(id)), (id) => !bookkeeping.has(id),
+    );
+    return { identities, bookkeeping };
+  };
+  const base = index(baseSnap), cand = index(candSnap);
+  const baseBook = base.bookkeeping, candBook = cand.bookkeeping;
+  const { pairs: idPairs } = pairEqualCounts(base.identities, cand.identities);
+
+  const baseFigures = new Map                                                ();
+  const candFigures = new Map                                                ();
+  const figures = (cache                    , snap                 , id        ) => {
+    let f = cache.get(id);
+    if (!f) { f = stageFigures(snap.stages.get(id) ); cache.set(id, f); }
+    return f;
+  };
+
+  const pairs              = idPairs
+    .map(({ baseId, candId })            => {
+      const b = figures(baseFigures, baseSnap, baseId), c = figures(candFigures, candSnap, candId);
+      const deltas = {}                                       ;
+      for (const metric of PAIR_DELTA_METRICS) {
+        const [bv, cv] = [b[metric], c[metric]];
+        deltas[metric] = { baseline: bv, candidate: cv, delta: bv != null && cv != null ? cv - bv : null };
+      }
+      return { pairId: `b${baseId}-c${candId}`, baseStageIds: [baseId], candStageIds: [candId], quality: 'exact', score: 1, deltas };
+    })
+    .sort((x, y) => x.baseStageIds[0] - y.baseStageIds[0] || x.candStageIds[0] - y.candStageIds[0]);
+
+  const pairedBase = new Set(pairs.flatMap((p) => p.baseStageIds));
+  const pairedCand = new Set(pairs.flatMap((p) => p.candStageIds));
+  const unmatchedOf = (snap                 , book             , paired             )           =>
+    [...snap.stages.keys()].filter((id) => !book.has(id) && !paired.has(id)).sort(ascending);
+
+  // Run time counts every attempt, the same figure the stage-pair deltas and the run totals use.
+  const runTime = (cache                    , snap                 , id        )         => figures(cache, snap, id).executorRunTime ?? 0;
+  let total = 0, matched = 0;
+  for (const id of baseSnap.stages.keys()) {
+    if (baseBook.has(id)) continue;
+    const t = runTime(baseFigures, baseSnap, id);
+    total += t;
+    if (pairedBase.has(id)) matched += t;
+  }
+  for (const id of candSnap.stages.keys()) {
+    if (candBook.has(id)) continue;
+    const t = runTime(candFigures, candSnap, id);
+    total += t;
+    if (pairedCand.has(id)) matched += t;
+  }
+
+  return {
+    pairs,
+    unmatched: { baseStageIds: unmatchedOf(baseSnap, baseBook, pairedBase), candStageIds: unmatchedOf(candSnap, candBook, pairedCand) },
+    replanned: [],
+    bookkeepingStageIds: { baseStageIds: [...baseBook].sort(ascending), candStageIds: [...candBook].sort(ascending) },
+    runtimeCoverage: total > 0 ? matched / total : null,
+  };
+}

package/vendor-core/stage-identity.js

@@ -0,0 +1,112 @@
+import { cyrb53 } from './string-hash.js';
+import { planNodesOfStage } from './stage-plan-nodes.js';
+                                                  
+                                                             
+
+// The stage-identity recipe, parameterized by the detail normalizer. `stageIdentity` (the frozen
+// exact key, run-comparison.ts) runs it with `normalizeDetail`; the aligner's `comparisonIdentity`
+// runs it with the comparison normalizer. One recipe, so the two keys can differ only in how a
+// detail string is normalized.
+
+                                                          
+
+// Replace run-varying tokens (digit runs, long hex ids) with a stable marker so
+// the same logical stage across two runs normalizes to one identity.
+export function normalizeStageName(name        )         {
+  return String(name)
+    .toLowerCase()
+    .replace(/\b[0-9a-f]{8,}\b/g, '#') // hex ids/uuids first (they contain digits)
+    .replace(/\d+/g, '#')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+// Bottom-up, order-independent structural identity of a resolved plan tree:
+// each node folds its normalized name/detail with its children's digests
+// (children sorted, so AQE picking a different broadcast side still matches),
+// so two plans collide only when their whole shape and every node's detail
+// agree. `normalizeDetail` strips the run-to-run noise (expr ids, `plan_id=`,
+// codegen numbers, AQE build-side choice, commutative-operand order). Each
+// node folds to a fixed-length cyrb53 digest (JSON.stringify-encoded, so
+// detail text containing `<`/`>`/`{`/`,` can't collide two different plans)
+// instead of embedding full child identity strings, which would re-escape
+// every level below and blow identity size up to ~2^depth on the 20-60+
+// operator-deep plans real Spark produces.
+function planTreeIdentity(root                             , normalizeDetail                  )                {
+  if (!root) return null;
+  function visit(node          )         {
+    const childDigests = (node.children ?? []).map(visit).sort();
+    return cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? ''), childDigests]));
+  }
+  return visit(root);
+}
+
+// Plan identity for the stage's SQL execution, scoped to only the plan nodes
+// this stage actually ran (`node.stageIds`), not the whole tree: two stages
+// sharing one SQL execution (e.g. a self-join's two Exchange stages) otherwise
+// collapse onto one identity regardless of which part of the plan each ran.
+// Falls back to the coarser whole-tree identity when the stage has no
+// attributed nodes (hand-built snapshots without `stageIds`, or unmatched
+// accumulables).
+function sqlNodeIdentity(
+  stage       , snapshot                              , normalizeDetail                  , nodes             ,
+)         {
+  const execId = stage.sqlExecutionId;
+  if (execId == null) return '';
+  const root = snapshot.sql.get(execId)?.planTree ?? null;
+  if (!root) return '';
+  const fingerprints = (nodes ?? planNodesOfStage(stage, snapshot.sql))
+    .map((node) => JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
+  if (fingerprints.length === 0) return planTreeIdentity(root, normalizeDetail) ?? '';
+  return cyrb53(JSON.stringify(fingerprints.sort()));
+}
+
+/** `nodes`: the stage's attributed plan nodes when the caller already has them (`planNodesOfStage`),
+ * so one walk of the plan serves several readers. */
+export function stageIdentityWith(
+  stage       , snapshot                              , normalizeDetail                  , nodes             ,
+)         {
+  return normalizeStageName(stage.name ?? '') + '§' + sqlNodeIdentity(stage, snapshot, normalizeDetail, nodes);
+}
+
+export function identityIndexWith(
+  snapshot                 , identityOf                                      , only                          = () => true,
+)                        {
+  const byIdentity = new Map                  ();
+  for (const [id, stage] of snapshot.stages) {
+    if (!only(id)) continue;
+    const key = identityOf(stage, id);
+    const ids = byIdentity.get(key);
+    if (ids) ids.push(id);
+    else byIdentity.set(key, [id]);
+  }
+  return byIdentity;
+}
+
+/** Pairs the stages of two identity indexes. An identity colliding equally on both sides has no
+ * genuine ambiguity about *count*, so its stages pair off positionally by sorted id rather than
+ * being dropped. This is exact when the identity actually distinguishes stages (e.g.
+ * self-comparing a run: every stage matches itself). It's a best-effort guess when the identity is
+ * coarse (no SQL/attribution) and the two sides are genuinely different runs -- two unrelated
+ * same-named stages could get cross-paired. Accepted tradeoff: dropping them instead would also
+ * sacrifice the exact-self-comparison case, which matters more. An identity with different counts
+ * per side pairs nothing. Deterministic and symmetric: identities visit in sorted order. */
+export function pairEqualCounts(
+  baseIdx                       , candIdx                       ,
+)                                                                                                         {
+  const pairs                                                              = [];
+  const matchedIdentities = new Set        ();
+  for (const identity of [...baseIdx.keys()].sort()) {
+    const candIds = candIdx.get(identity);
+    if (!candIds) continue;
+    const baseIds = baseIdx.get(identity) ;
+    if (baseIds.length !== candIds.length) continue;
+    const sortedBaseIds = [...baseIds].sort((a, b) => a - b);
+    const sortedCandIds = [...candIds].sort((a, b) => a - b);
+    for (let i = 0; i < sortedBaseIds.length; i++) {
+      pairs.push({ identity, baseId: sortedBaseIds[i], candId: sortedCandIds[i] });
+    }
+    matchedIdentities.add(identity);
+  }
+  return { pairs, matchedIdentities };
+}

package/vendor-core/stage-plan-nodes.js

@@ -0,0 +1,18 @@
+                                                                
+
+/** The plan nodes a stage actually ran: its SQL execution's resolved plan tree, filtered to the
+ * nodes attributed to it (`node.stageIds`). Empty when the stage has no SQL execution, the
+ * execution has no resolved plan, or no node is attributed to it. The one stage-to-plan mapping:
+ * stageIdentity's fingerprint and the Python-stage check both read it. */
+export function planNodesOfStage(stage                                      , sql                           )             {
+  const execId = stage.sqlExecutionId;
+  if (execId == null) return [];
+  const root = sql.get(execId)?.planTree ?? null;
+  if (!root) return [];
+  const nodes             = [];
+  (function collect(node          )       {
+    if (node.stageIds?.includes(stage.id)) nodes.push(node);
+    for (const child of node.children ?? []) collect(child);
+  })(root);
+  return nodes;
+}

package/vendor-core/stage-quantiles.js

@@ -56,6 +56,7 @@ const TASK_FIELD_PROPS = TASK_FIELD_DESCRIPTORS.map((d) => d.prop);
                           
                      
                       
+                               
  
 
 export function finalizeStage(
@@ -124,9 +125,11 @@ export function finalizeStage(
     acc.executorCpuTime += t.executorCpuTime;
     acc.inputBytes += t.inputBytes;
     acc.outputBytes += t.outputBytes;
+    if (t.outputRecords != null) acc.outputRecords = (acc.outputRecords ?? 0) + t.outputRecords;
     for (let i = 0; i < TASK_FIELD_PROPS.length; i++) buf.push(t[TASK_FIELD_PROPS[i]]);
   }
   stage.taskCount = taskCount;
+  stage.peakExecutionMemoryMax = peakExecutionMemoryMax;
   stage.failedTasks = failedTasks;
   stage.speculativeTasks = speculativeTasks;
   stage.taskAttempts = null; // no longer needed after finalize, freeing memory
@@ -201,6 +204,7 @@ export function finalizeStage(
   delete data.failureDetails; // internal-only intern table, summarized by failureGroups
   delete data.speculativeWinners; // internal-only late-TaskEnd pairing state, kept worker-side
   delete data.lateSpeculationWaste;
+  delete data.stageAttemptId;
 
   return { type: 'stage', data };
 }

package/vendor-core/types.js

@@ -57,6 +57,24 @@
  
                                                                                                  
 
+                                     
+                    
+                      
+                         
+                          
+                          
+                    
+                             
+                           
+                           
+                            
+                     
+                      
+                               
+                                 
+                            
+ 
+
                         
               
                 
@@ -70,6 +88,20 @@
                        
                       
                        
+                                                                                                  
+                                
+                                                                                        
+                                                                    
+                         
+                               
+                                                                                               
+                                                                                       
+                                              
+                                                                                                
+                                                                                            
+                                                                                                  
+                             
+                                              
                             
                           
                           
@@ -124,6 +156,11 @@
                                
              
                              
+                       
+                     
+                           
+                                                                                                        
+                            
                          
  
                                     
@@ -263,6 +300,10 @@
                          
                                       
                                                     
+                                                                               
+                        
+                                                                                
+                                     
  
 
                                                                    
@@ -272,6 +313,9 @@
                                  
                 
                      
+                                                                                         
+                                                                                    
+              
  
 
                                                                                             
@@ -288,11 +332,20 @@
                                                   
                                        
                             
+                                                                                               
+                                                                                               
+                                                                                             
+                                                                                                 
+                                                                                               
+                                                                                   
+                                                                                                                                  
+                                                                     
+                                                    
  
 
 // `Finding` is a union discriminated on `type`, one member per emitted finding type: see
 // finding-types.ts for each detector's shape and which of its fields are public evidence.
-                                                                                                                  
+