sparkforensics-cli 0.1.0 → 0.2.0

package/vendor-core/event-handlers.js

@@ -24,12 +24,9 @@ import { computeRunAggregates } from './run-aggregates.js';
                                                                                         
                  
 
-// ---------------------------------------------------------------------------
-// Internal parser-state shapes. These describe the real runtime objects the
-// handlers below build and mutate; they are intentionally not the same as
-// the public AppModel types in types.ts (e.g. Stage), which describe the
-// *posted* message shape after finalizeStage/appMessage reshape things.
-// ---------------------------------------------------------------------------
+// Internal parser-state shapes: the real runtime objects the handlers build and mutate, not the
+// public AppModel types in types.ts (which describe the posted message shape after
+// finalizeStage/appMessage reshape things).
 
                             
              
@@ -50,9 +47,8 @@ import { computeRunAggregates } from './run-aggregates.js';
                                            
                                         
                             
-                                                                              
-                                                                           
-                                     
+                                                                                                
+                          
                          
  
 
@@ -70,12 +66,31 @@ import { computeRunAggregates } from './run-aggregates.js';
 
                                                                                       
 
-// One accumulated task-attempt record, keyed by `<stageAttemptId>:<index>` (or
-// a unique Symbol when the raw event has no Index). Matches accumulateTask's
-// real built shape (event-handlers.js:169-189).
+                                   
+                        
+                        
+               
+                     
+                        
+                      
+                     
+                       
+ 
+
+// Cap for both StageRecord.retryTaskSamples (populated here, live during ingest) and
+// stage.failedTaskSamples (populated in stage-quantiles.ts's finalizeStage): a starting judgment
+// call, not empirically validated against a pathological case. Kept as a separate constant in each
+// file rather than a shared import, to avoid a circular value import between the two modules
+// (event-handlers.ts already imports finalizeStage FROM stage-quantiles.ts).
+const MAX_TASK_SAMPLES = 20;
+
+// One accumulated task-attempt record, keyed by `<stageAttemptId>:<index>` (or a unique Symbol
+// when the raw event has no Index).
                       
                    
                   
+                        
+                        
                      
                      
                         
@@ -121,6 +136,7 @@ import { computeRunAggregates } from './run-aggregates.js';
                                       
                                     
                                                         
+                                       
                        
                          
                              
@@ -136,8 +152,7 @@ import { computeRunAggregates } from './run-aggregates.js';
                      
                                   
                                       
-                                                                         
-                                                                             
+                                                                                                  
                                                                           
                              
  
@@ -160,19 +175,85 @@ import { computeRunAggregates } from './run-aggregates.js';
                                             
                                  
  
+
+// Splits decompressed byte chunks into NDJSON lines. Each chunk is decoded whole in one
+// TextDecoder.decode call, newlines located by scanning raw bytes (indexOf(0x0A)), and each line
+// is a substring of the `pending + text` concatenation.
+//
+// Why (profiled on a 138MB / 3.5GB-decompressed log, decode phase, each line JSON.parsed to force
+// materialization): per-line decode (227k calls) ~57s vs whole-chunk decode (~36k calls) ~7s;
+// scanning the decoded UTF-16 string for newlines costs 130-450s while scanning raw bytes ~7s;
+// the line MUST be a substring of the freshly concatenated string (V8 flattens once per chunk),
+// substrings of raw decode() output cost ~4x more. Combined ~14s, byte-identical output.
+//
+// 0x0A can't appear inside a UTF-8 multibyte sequence, so byte-scanning for it is UTF-8 safe. A
+// newline's byte offset equals its char offset only when the chunk is 1:1 byte<->char, detected by
+// text.length === buffer.length (the all-ASCII fast path); otherwise a byte walk recovers char
+// offsets. `{ stream: true }` reassembles a character split across the chunk boundary.
+const NEWLINE = 0x0a;
 
 export function buildChunkDecoder() {
   const decoder = new TextDecoder('utf-8');
+  // Decoded partial line after the last newline, carried to the next chunk. A character split
+  // across the boundary is reassembled by the streaming decoder, so this is already-decoded.
   let pending = '';
+
   return {
     decode(buffer            )           {
-      const text = pending + decoder.decode(buffer, { stream: true });
-      const lines = text.split('\n');
-      pending = lines.pop() ?? '';
-      return lines.filter((l) => l.length > 0);
+      const lines           = [];
+      const text = decoder.decode(buffer, { stream: true });
+      const full = pending === '' ? text : pending + text;
+      const base = pending.length;
+      const len = buffer.length;
+      // Char offset in `full` of the current line's start. Zero-length lines
+      // (`cut === start`) are dropped to match a `.filter(l => l.length)`.
+      let start = 0;
+
+      if (text.length === len) {
+        // 1:1 byte<->char: a newline's byte offset is its char offset (+ base).
+        let nl = buffer.indexOf(NEWLINE, 0);
+        while (nl !== -1) {
+          const cut = base + nl;
+          if (cut > start) lines.push(full.substring(start, cut));
+          start = cut + 1;
+          nl = buffer.indexOf(NEWLINE, nl + 1);
+        }
+      } else {
+        // Some byte does not map 1:1. Walk the bytes counting UTF-16 units to
+        // turn each newline's byte offset into a char offset in `full`.
+        let unit = base;
+        let bpos = 0;
+        // A char whose lead byte was in the previous chunk arrives here as
+        // leading continuation bytes; the streaming decoder emits it as text[0].
+        // Count it once (a surrogate pair is two units), then skip its bytes.
+        if (len > 0 && (buffer[0] & 0xc0) === 0x80) {
+          const c0 = text.charCodeAt(0);
+          unit += c0 >= 0xd800 && c0 <= 0xdbff ? 2 : 1;
+          while (bpos < len && (buffer[bpos] & 0xc0) === 0x80) bpos++;
+        }
+        let nl = buffer.indexOf(NEWLINE, bpos);
+        while (nl !== -1) {
+          for (let i = bpos; i < nl; i++) {
+            const b = buffer[i];
+            if (b < 0x80) unit++; // ASCII
+            else if (b >= 0xf0) unit += 2; // 4-byte lead -> surrogate pair
+            else if (b >= 0xc0) unit++; // 2/3-byte lead -> one unit
+            // continuation byte (0x80..0xBF) -> zero units
+          }
+          if (unit > start) lines.push(full.substring(start, unit));
+          unit++; // the '\n' itself
+          start = unit;
+          bpos = nl + 1;
+          nl = buffer.indexOf(NEWLINE, bpos);
+        }
+      }
+
+      pending = start < full.length ? full.substring(start) : '';
+      return lines;
     },
     flush()           {
-      const last = pending.trim();
+      // Flush any incomplete trailing character the streaming decoder still holds.
+      const last = (pending + decoder.decode()).trim();
       pending = '';
       return last ? [last] : [];
     },
@@ -209,13 +290,9 @@ export function createState()              {
   };
 }
 
-// Normalize the `Spark Properties` payload of SparkListenerEnvironmentUpdate.
-// Modern Spark emits an object of string key→value; older logs use an array of
-// [key, value] pairs. Both collapse to a plain string→string map. The schema
-// (EnvironmentUpdateEventSchema) tolerates raw values that are a bare JSON
-// number or boolean instead of a string, so those are coerced to their
-// string form here, keeping this function's `Record<string, string>`
-// contract intact for every downstream reader of `app.config`.
+// Normalize the `Spark Properties` payload of SparkListenerEnvironmentUpdate. Modern Spark emits
+// an object of key->value; older logs use [key, value] pairs. Both collapse to a string->string
+// map; the schema tolerates bare number/boolean values, coerced to string to keep the contract.
                                                     
 export function normalizeSparkProperties(
   props                                                                                        
@@ -234,10 +311,9 @@ export function normalizeSparkProperties(
   return map;
 }
 
-// Parse a Spark memory-size string to MiB. Spark's JVM-memory configs
-// (spark.{executor,driver}.memory[Overhead]) use bytesConf(ByteUnit.MiB), so a
-// bare number ("10") means MiB. A k/m/g/t suffix sets the unit (an optional
-// trailing "b", as in "gb", is redundant). A lone trailing "b" ("10b") means bytes.
+// Parse a Spark memory-size string to MiB. Spark's JVM-memory configs use bytesConf(ByteUnit.MiB),
+// so a bare number means MiB. A k/m/g/t suffix sets the unit (trailing "b" redundant); a lone "b"
+// ("10b") means bytes.
 export function parseSparkMemoryMB(value         )                {
   if (value == null) return null;
   const m = String(value).trim().toLowerCase().match(/^([\d.]+)\s*([kmgt]?)(b?)$/);
@@ -253,9 +329,8 @@ export function parseSparkMemoryMB(value         )                {
   }
 }
 
-// Derive an allocated-resource summary (executor/driver memory + cores, plus
-// dynamic-allocation / shuffle-service flags) from the Spark config map.
-// Absent keys degrade to null rather than guessed defaults.
+// Derive an allocated-resource summary from the Spark config map. Absent keys degrade to null,
+// not guessed defaults.
 export function extractResources(config                                           )                   {
   const cfg = config ?? {};
   const int = (k        ) => {
@@ -287,11 +362,8 @@ export function extractResources(config
   };
 }
 
-// Convert the live rddInfo Map (whose entries carry a mutable `stageIds` Set
-// used for bookkeeping across StageSubmitted events) into a postable copy
-// with `stageIds` as a sorted array, used at every 'app' message post site
-// so the main thread never receives a live Set it can't structured-clone
-// meaningfully or that could keep mutating after being posted.
+// Convert the live rddInfo Map (with a mutable stageIds Set) into a postable copy with stageIds
+// as a sorted array, so the main thread never receives a live Set that could keep mutating after post.
 function snapshotRddInfo(rddInfo                            )                                     {
   const out = new Map                               ();
   for (const [id, r] of rddInfo) {
@@ -305,16 +377,10 @@ function snapshotEvidenceInputs(state             )                 {
 }
 
 /**
- * Snapshots the current evidence counters and posts an `app` message. This is
- * called from `SparkListenerApplicationStart`, `SparkListenerEnvironmentUpdate`,
- * and `SparkListenerApplicationEnd`, so every streaming `app` message carries a
- * counter snapshot taken at that moment. Only the terminal snapshot emitted by
- * `emitParseCompletion` is authoritative; mid-parse snapshots are partial and
- * must not be used for final evidence-availability conclusions.
- *
- * Only ever called once `state.app` has been assigned (from startApplication,
- * or from updateEnvironment/the ApplicationEnd case which both guard on
- * `state.app` truthiness before calling this), hence the non-null assertion.
+ * Snapshots the evidence counters and posts an `app` message. Called from ApplicationStart,
+ * EnvironmentUpdate, and ApplicationEnd, so every streaming `app` message carries a snapshot at
+ * that moment. Only emitParseCompletion's terminal snapshot is authoritative; mid-parse ones are
+ * partial. Only called once state.app is assigned (all callers guard on it), hence the non-null assertion.
  */
 function appMessage(state             )                                                 {
   const evidenceInputs = snapshotEvidenceInputs(state);
@@ -325,14 +391,19 @@ function appMessage(state             )
   };
 }
 
+function taskRecordToSample(t            )                   {
+  return {
+    taskId: t.taskId, attemptNumber: t.attemptNumber, host: t.host, executorId: t.executorId,
+    reason: t.reason, peakExecMem: t.peakExecMem, memSpilled: t.memSpilled, shuffleWrite: t.shuffleWrite,
+  };
+}
+
 export function accumulateTask(event                                    , state             )       {
   const stageId = event['Stage ID'];
   const stage = state.stages.get(stageId);
   if (!stage) return null;
-  // Late TaskEnd for a stage whose StageCompleted already arrived and freed
-  // taskAttempts (finalizeStage), same out-of-order tolerance as
-  // SparkListenerStageExecutorMetrics above; the task's stats are already
-  // baked into the finalized stage and don't need to be re-added.
+  // Late TaskEnd for a stage whose StageCompleted already freed taskAttempts (finalizeStage): its
+  // stats are already baked into the finalized stage, don't re-add.
   if (stage.taskAttempts === null) return null;
 
   state.evidenceInputs.taskRecords++;
@@ -343,10 +414,8 @@ export function accumulateTask(event                                    , state
     state.taskAccumStages.get(acc.ID) .add(stageId);
   }
 
-  // 'Task Info' itself and its Failed/Killed/Speculative fields are all
-  // optional in the schema (real event-log variants omit them); Partial<>
-  // lets the {} fallback (an absent 'Task Info') type-check while every
-  // read below still defaults via `??`/`||`.
+  // 'Task Info' and its Failed/Killed/Speculative fields are optional in the schema; Partial<>
+  // lets the {} fallback type-check while reads below default via ??/||.
                                                                                            
   const info              = event['Task Info'] ?? {};
   const m = event['Task Metrics'] ?? {};
@@ -360,6 +429,8 @@ export function accumulateTask(event                                    , state
 
   const record             = {
     duration, failed,
+    taskId: info['Task ID'] ?? null,
+    attemptNumber: info['Attempt Number'] ?? 0,
     launchTime: info['Launch Time'] ?? 0,
     finishTime: info['Finish Time'] ?? 0,
     reason: event['Task End Reason']?.['Reason'] ?? null,
@@ -380,9 +451,8 @@ export function accumulateTask(event                                    , state
     outputBytes: out['Bytes Written'] ?? 0,
   };
 
-  // Dedupe only when Index is present (always true for real Spark event logs).
-  // Without it, every event is treated as a distinct task: preserves legacy
-  // fixture behavior for tests that omit Index.
+  // Dedupe only when Index is present (always true for real logs). Without it every event is a
+  // distinct task, preserving behavior for fixtures that omit Index.
   const index = info['Index'];
   const key                  = index != null ? `${event['Stage Attempt ID'] ?? 0}:${index}` : Symbol('no-index');
   const existing = stage.taskAttempts.get(key);
@@ -390,21 +460,18 @@ export function accumulateTask(event                                    , state
   if (!existing) {
     stage.taskAttempts.set(key, record);
   } else if (existing.failed && !record.failed) {
-    // A retry succeeded where the earlier attempt failed: the earlier
-    // attempt's time was wasted work, not a metric to fold twice.
-    //
-    // Cause classification: Spark only marks the speculative COPY's own
-    // TaskInfo.Speculative as true, never the original it raced against,
-    // so checking either side of the comparison (not just the discarded
-    // record) is required to catch both "original loses to its speculative
-    // twin" and "speculative twin loses to the original" symmetrically,
-    // regardless of arrival order.
+    // A retry succeeded where the earlier attempt failed: the earlier attempt's time was wasted.
+    // Spark marks only the speculative COPY's Speculative flag, never the original it raced, so
+    // check both sides to catch "original loses to twin" and "twin loses to original" either way.
     if (existing.speculative || record.speculative) {
       stage.speculationWasteMs += existing.duration;
       stage.speculationWastedAttempts++;
     } else {
       stage.retryWasteMs += existing.duration;
       stage.wastedAttempts++;
+      if (stage.retryTaskSamples.length < MAX_TASK_SAMPLES) {
+        stage.retryTaskSamples.push(taskRecordToSample(existing));
+      }
     }
     stage.taskAttempts.set(key, record);
   } else {
@@ -416,6 +483,9 @@ export function accumulateTask(event                                    , state
     } else {
       stage.retryWasteMs += record.duration;
       stage.wastedAttempts++;
+      if (stage.retryTaskSamples.length < MAX_TASK_SAMPLES) {
+        stage.retryTaskSamples.push(taskRecordToSample(record));
+      }
     }
   }
 
@@ -427,11 +497,24 @@ export function resolvePlanTree(
   accumMap                     ,
   taskAccumStages                          ,
   executionStageIds                         ,
+  executionId         ,
 )           {
   const seen = new WeakSet               ();
-  const nodeMap = new WeakMap                         (); // sparkPlanInfoNode → resolved PlanNode
-
-  function makeNode(info               )           {
+  const nodeMap = new WeakMap                         ();
+  let nextId = 0;
+  // Prefixed with the owning SQL execution so PlanNode.id is globally unique,
+  // not just unique within this one call: two executions' trees can each
+  // produce a node named "n1", and a consumer keying off id alone (e.g. a
+  // findings-by-nodeId index) must be able to tell them apart without also
+  // threading execution scope through separately. No executionId (e.g. the
+  // legacy unit tests below) falls back to the old unprefixed "n0", "n1", ...
+  // shape.
+  const idPrefix = executionId != null ? `e${executionId}:` : '';
+
+  function computeMetricsAndStageIds(info               )   
+                                                                    
+                        
+    {
     const metrics = (info.metrics ?? []).reduce                                                        ((acc, m) => {
       if (m.accumulatorId !== undefined && accumMap.has(m.accumulatorId)) {
         acc.push({ name: m.name, value: accumMap.get(m.accumulatorId) , metricType: m.metricType });
@@ -439,27 +522,63 @@ export function resolvePlanTree(
       return acc;
     }, []);
 
-    // Union the stages where this node's accumulator IDs were actually
-    // observed running (Task 2's data), then clip to this SQL execution's own
-    // stage universe (Task 3's data): an accumulator ID can occasionally
-    // belong to a different execution's stages (e.g. a reused subquery),
-    // and without the clip that would misattribute another execution's work.
     const stageIdSet = new Set        ();
     for (const m of info.metrics ?? []) {
       if (m.accumulatorId === undefined) continue;
       const stages = taskAccumStages.get(m.accumulatorId);
       if (!stages) continue;
+      // Clip to executionStageIds (this execution's own stage set) rather than trusting
+      // taskAccumStages wholesale: an accumulator id can be shared with another execution's
+      // stages (e.g. a reused subquery re-executes the same accumulator-tagged operator under
+      // a different SQL execution id), and without this clip a plan node here would pick up
+      // stage ids that actually belong to that other execution's work.
       for (const sid of stages) {
         if (executionStageIds && executionStageIds.has(sid)) stageIdSet.add(sid);
       }
     }
+    return stageIdSet.size > 0 ? { metrics, stageIds: [...stageIdSet].sort((a, b) => a - b) } : { metrics };
+  }
 
-    const node           = { name: info.nodeName, detail: info.simpleString ?? '', metrics, children: [] };
-    if (stageIdSet.size > 0) node.stageIds = [...stageIdSet].sort((a, b) => a - b);
+  function makeNode(info               , children            )           {
+    const { metrics, stageIds } = computeMetricsAndStageIds(info);
+    const node           = { id: `${idPrefix}n${nextId++}`, name: info.nodeName, detail: info.simpleString ?? '', metrics, children };
+    if (stageIds) node.stageIds = stageIds;
     return node;
   }
 
-  // DFS to collect nodes in discovery order
+  // Only a node whose raw name is exactly Exchange/BroadcastExchange gets
+  // split; ReusedExchange and any other exchange-family variant stay a
+  // single node (confirmed against real event logs: ReusedExchange carries
+  // no metrics of its own, it references an already-materialized Exchange
+  // elsewhere). Runs on the raw name because no PlanNode/exchangeRole
+  // exists yet at this point in construction.
+  function makeExchangeSplit(info               , children            )           {
+    const { metrics, stageIds } = computeMetricsAndStageIds(info);
+    // write = producer side: keeps the real metrics (e.g. "data size", the
+    // shuffle write metrics) and the original children.
+    const write           = {
+      id: `${idPrefix}n${nextId++}`,
+      name: info.nodeName,
+      detail: '',
+      metrics,
+      children,
+      exchangeRole: 'write',
+    };
+    if (stageIds) write.stageIds = stageIds;
+    // read = consumer side: keeps the original detail (plan-summary.ts's
+    // partitioning-key regex needs it), no metrics of its own.
+    const read           = {
+      id: `${idPrefix}n${nextId++}`,
+      name: info.nodeName,
+      detail: info.simpleString ?? '',
+      metrics: [],
+      children: [write],
+      exchangeRole: 'read',
+    };
+    if (stageIds) read.stageIds = stageIds;
+    return read;
+  }
+
   const stack                  = [rootInfo];
   const order                  = [];
 
@@ -473,24 +592,24 @@ export function resolvePlanTree(
     }
   }
 
-  // Build nodes bottom-up (reverse discovery = children before parents)
   for (let i = order.length - 1; i >= 0; i--) {
     const info = order[i];
-    const node = makeNode(info);
-    nodeMap.set(info, node);
+    const children             = [];
     for (const child of (info.children ?? [])) {
       const childNode = nodeMap.get(child);
-      if (childNode) node.children.push(childNode);
+      if (childNode) children.push(childNode);
     }
+    const node = info.nodeName === 'Exchange' || info.nodeName === 'BroadcastExchange'
+      ? makeExchangeSplit(info, children)
+      : makeNode(info, children);
+    nodeMap.set(info, node);
   }
 
   return nodeMap.get(rootInfo) ;
 }
 
-// Maps SparkListenerStageExecutorMetrics' "Executor Metrics" field names
-// (Spark's ExecutorMetricType constants) to our camelCase data-model names.
-// Only listed fields are captured: anything else is ignored (forward-compat
-// with future Spark ExecutorMetricType additions).
+// Maps SparkListenerStageExecutorMetrics' "Executor Metrics" field names to our camelCase names.
+// Only listed fields are captured (forward-compat with future Spark ExecutorMetricType additions).
 const EXECUTOR_METRIC_FIELD_MAP                         = {
   JVMHeapMemory: 'jvmHeapMemory', JVMOffHeapMemory: 'jvmOffHeapMemory',
   OnHeapExecutionMemory: 'onHeapExecutionMemory', OffHeapExecutionMemory: 'offHeapExecutionMemory',
@@ -524,9 +643,8 @@ export function updateEnvironment(event
   state.evidenceInputs.environmentUpdates++;
   const config = normalizeSparkProperties(event['Spark Properties']);
   const resources = extractResources(config);
-  // EnvironmentUpdate normally precedes ApplicationStart: stash the config
-  // so ApplicationStart can attach it. If it arrives after (a mid-run
-  // update), apply it live and re-post the app so the main thread refreshes.
+  // EnvironmentUpdate normally precedes ApplicationStart: stash the config so ApplicationStart can
+  // attach it. If it arrives after (a mid-run update), apply live and re-post the app.
   if (state.app) {
     state.app.config = config;
     state.app.resources = resources;
@@ -597,6 +715,7 @@ export function submitStage(event                                           , st
     failureReasons: new Map(),
     stageFailureReason: null,
     taskAttempts: new Map(),
+    retryTaskSamples: [],
     retryWasteMs: 0,
     wastedAttempts: 0,
     speculationWasteMs: 0,
@@ -631,11 +750,9 @@ export function mergeStageRddInfo(
         replication: sl['Replication'] ?? 1,
       },
       numPartitions: rdd['Number of Partitions'] ?? 0,
-      // Merge forward, don't overwrite: an RDD being cached for the first
-      // time in THIS stage legitimately reports 0 here (Spark's snapshot
-      // reflects BlockManager state at submission time). If a later
-      // resubmission arrives before the cache numbers are re-observed,
-      // keep the last real value instead of regressing to 0/em-dash.
+      // Merge forward, don't overwrite: an RDD cached for the first time in THIS stage legitimately
+      // reports 0 (snapshot reflects BlockManager state at submission). Keep the last real value on
+      // a resubmission instead of regressing to 0.
       numCachedPartitions: rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
       memorySize: rdd['Memory Size'] || prev?.memorySize || 0,
       diskSize: rdd['Disk Size'] || prev?.diskSize || 0,
@@ -644,13 +761,9 @@ export function mergeStageRddInfo(
   }
 }
 
-// Spark's EventLoggingListener logs these AFTER the stage's
-// SparkListenerStageCompleted, so finalizeStage has already posted
-// (structured-cloned) the stage with an empty executorMetrics Map by the
-// time they arrive. We keep accumulating into the worker-side stage here,
-// then re-post all populated maps once via `stageExecutorMetrics` just
-// before `done` (see collectStageExecutorMetrics) so the main-thread
-// stages are patched before analyze() runs.
+// Spark logs these AFTER SparkListenerStageCompleted, so finalizeStage has already posted the
+// stage with an empty executorMetrics Map. Keep accumulating worker-side, then re-post all maps
+// once via `stageExecutorMetrics` just before `done` so main-thread stages are patched before analyze().
 export function recordStageExecutorMetrics(event                                                 , state             )       {
   const stage = state.stages.get(event['Stage ID']);
   if (!stage) return null;
@@ -697,7 +810,9 @@ export function endSqlExecution(
   }
 
   const accumMap = state.accumState.get(event.executionId) ?? new Map                ();
-  const planTree = resolvePlanTree(planInfo, accumMap, state.taskAccumStages, state.sqlExecStages.get(event.executionId));
+  const planTree = resolvePlanTree(
+    planInfo, accumMap, state.taskAccumStages, state.sqlExecStages.get(event.executionId), event.executionId,
+  );
   state.accumState.delete(event.executionId);
 
   state.evidenceInputs.resolvedSqlPlans++;
@@ -723,13 +838,9 @@ export function applyAdaptiveExecutionUpdate(
   if (event.sparkPlanInfo != null) exec.sparkPlanInfo = event.sparkPlanInfo;
   if (event.physicalPlanDescription != null) exec.physicalPlanDescription = event.physicalPlanDescription;
   exec.hadAdaptiveUpdate = true;
-  // Re-emit a 'sql' message (same shape startSqlExecution posts) so the
-  // browser's structured-cloned appModel.sql copy actually sees the flip:
-  // without this, hadAdaptiveUpdate only ever reads true via collectRun's
-  // Node-path object aliasing (no postMessage clone in that path), never in
-  // the shipping worker. Shallow copy so the posted object isn't the same
-  // mutable reference the worker keeps mutating (mirrors endSqlExecution's
-  // `{ ...exec }` pattern above).
+  // Re-emit a 'sql' message so the browser's structured-cloned appModel.sql copy sees the flip:
+  // otherwise hadAdaptiveUpdate only reads true via collectRun's Node-path object aliasing, never
+  // in the shipping worker. Shallow copy so the posted object isn't the mutable reference the worker keeps mutating.
   return { type: 'sql', data: { ...exec } };
 }
 
@@ -793,12 +904,8 @@ export function processEvent(event            , state             )          {
       if (!stage) return null;
       stage.completedAt = info['Completion Time'] ?? 0;
       stage.stageFailureReason = info['Failure Reason'] ?? null;
-      // finalizeStage (Task 5, stage-quantiles.ts) deliberately keeps its
-      // `stage` parameter typed as a loose Record (see that module's own
-      // comment); StageRecord's real shape (more precise than that loose
-      // type, e.g. taskAttempts values carry string/boolean fields too, not
-      // just numbers) is bridged across that boundary with an explicit cast
-      // rather than re-litigating finalizeStage's own established signature.
+      // finalizeStage keeps its `stage` parameter typed as a loose Record (see that module); bridge
+      // StageRecord's more precise shape across that boundary with an explicit cast.
       return finalizeStage(
         id,
         stage                                                                                                      ,
@@ -835,8 +942,7 @@ export function processEvent(event            , state             )          {
   }
 }
 
-// Derived (not hand-maintained) from SparkEventSchema itself, so this set
-// can never drift from the discriminated union's real literal list.
+// Derived from SparkEventSchema itself, so this set can never drift from the union's literal list.
 const KNOWN_EVENT_TYPES                      = new Set(
   SparkEventSchema.options.map((option) => option.shape.Event.value)
 );
@@ -849,13 +955,9 @@ export function dispatchLine(line        , state             , emit
     state.skippedLines++;
     return;
   }
-  // A real Spark event log carries many event types this tool has never
-  // modeled (BlockManagerAdded, TaskStart, ExecutorMetricsUpdate, ...): that
-  // was always true pre-migration and is completely benign, so an `Event`
-  // value outside our 15 modeled literals is silently ignored exactly like
-  // before, never counted toward skippedLines. Only a value that IS one of
-  // the 15 known types but fails ITS schema is a genuine, valuable signal
-  // worth surfacing as a skipped line.
+  // A real Spark event log carries many event types this tool never modeled (BlockManagerAdded,
+  // TaskStart, ...): an `Event` outside our modeled literals is silently ignored, never counted as
+  // a skipped line. Only a KNOWN type that fails ITS schema is a genuine signal worth surfacing.
   const eventType = (parsed                              )?.Event;
   if (typeof eventType !== 'string' || !KNOWN_EVENT_TYPES.has(eventType)) {
     return;
@@ -869,11 +971,9 @@ export function dispatchLine(line        , state             , emit
   try {
     msg = processEvent(result.data, state);
   } catch (err) {
-    // processEvent's switch is exhaustive over the schema-validated union
-    // (assertNever in the default case), so this should be unreachable for
-    // any event that made it this far. If it ever fires, it's a genuine bug
-    // in a handler, not malformed input; log it distinctly so it doesn't
-    // masquerade as an ordinary skipped-line (malformed/invalid data) count.
+    // processEvent's switch is exhaustive over the schema-validated union (assertNever default), so
+    // this is unreachable for any event that got this far. If it fires it's a handler bug, not
+    // malformed input; log it distinctly so it doesn't masquerade as an ordinary skipped-line count.
     console.error('processEvent threw for an already-validated event:', err);
     state.skippedLines++;
     return;
@@ -881,10 +981,8 @@ export function dispatchLine(line        , state             , emit
   if (msg) emit(msg);
 }
 
-// Gather every stage's post-completion-populated executorMetrics for a single
-// re-post just before `done` (see the SparkListenerStageExecutorMetrics case
-// for why the per-stage message posted at completion is empty). Returns a
-// Map<stageId, Map<execId, metrics>>; empty when the log had no
+// Gather every stage's post-completion-populated executorMetrics for a single re-post just before
+// `done` (the per-stage message posted at completion is empty). Empty when the log had no
 // spark.eventLog.logStageExecutorMetrics data.
 export function collectStageExecutorMetrics(state             )                                                   {
   const out = new Map                                             ();