sparkforensics-server 0.6.0 → 0.8.1

package/vendor-core/event-handlers.js

@@ -7,6 +7,7 @@ import {
   JobEndEventSchema,
   StageSubmittedEventSchema,
   StageExecutorMetricsEventSchema,
+  StageCompletedEventSchema,
   TaskEndEventSchema,
   SqlExecutionStartEventSchema,
   SqlAdaptiveExecutionUpdateEventSchema,
@@ -15,6 +16,7 @@ import {
   ExecutorAddedEventSchema,
   ExecutorRemovedEventSchema,
   BlockUpdatedEventSchema,
+  MAX_ACCUMULABLES_PER_TASK,
                   
                      
 } from './event-schemas.js';
@@ -24,6 +26,8 @@ import { finalizeStage } from './stage-quantiles.js';
 import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
 import { computeRunAggregates } from './run-aggregates.js';
 import { parseSparkMemoryMB } from './spark-memory.js';
+import { effectiveSparkConf } from './spark-conf.js';
+import { DRIVER_EXECUTOR_ID } from './executor-peaks.js';
              
                                                                                                             
                  
@@ -51,6 +55,9 @@ import { parseSparkMemoryMB } from './spark-memory.js';
                                            
                                         
                             
+                                                                                                     
+                                                                                               
+                                              
                                                                                                 
                           
                          
@@ -130,6 +137,8 @@ const MAX_TASK_SAMPLES = 20;
                           
                           
                      
+                       
+                             
                       
                                                                                                 
                                
@@ -202,6 +211,8 @@ const MAX_TASK_SAMPLES = 20;
                            
                                                                                   
                             
+                                                                                                          
+                                           
                                                                                          
                                       
                                                                                                   
@@ -226,12 +237,21 @@ const MAX_TASK_SAMPLES = 20;
                                                                                   
                                    
                                                
+                                                                                             
+                                                                                  
+                                                       
                                       
                                         
                                                                                                  
                                                                       
                           
                                             
+                                                                                                 
+                                                                                  
+                                                   
+                                                                                          
+                                                                                               
+                                                           
                                  
                                                                                      
                                                                                    
@@ -241,22 +261,18 @@ const MAX_TASK_SAMPLES = 20;
  
 
 // Splits decompressed byte chunks into NDJSON lines. Each chunk is decoded whole in one streaming
-// TextDecoder.decode call (per-line decode was ~8x slower: 227k calls vs ~36k on a 138MB / 3.5GB
-// log), then newlines are found with String.prototype.indexOf on that fresh, flat decoded text.
+// TextDecoder.decode call (per-line decode is ~8x slower), then newlines are found with
+// String.prototype.indexOf on that fresh, flat decoded text.
 // A '\n' char is exactly the 0x0A byte (0x0A never occurs inside a UTF-8 multibyte sequence), and
 // `{ stream: true }` reassembles a character split across a chunk boundary, so no byte-to-UTF-16
 // offset mapping is needed. Only the first line of a chunk joins the carried-over `pending` text.
 //
-// Measured 2026-09-23 against the previous raw-byte-scan version (identical output): 3.5s -> 2.8s
-// on that 3.5GB all-ASCII log, 0.37s -> 0.18s on 93MB of synthetic 2/3/4-byte-heavy NDJSON.
-//
 // A SQL UI event's physicalPlanDescription value (see stripPlanDescription) that is still open
 // when a chunk ends is cut off the pending line, and the following bytes are dropped up to its
-// closing quote without being decoded. On the largest real log that value is 1.9 GB of the
-// 3.5 GB stream, so it is never decoded, joined or scanned as text. A chunk longer than
-// MAX_DECODE_SLICE is decoded in slices of that size, so the skip also applies within one chunk:
-// Node's native zstd emits whole frames, up to 39 MB on that log, which held all but 51 MB of the
-// value inside a single chunk.
+// closing quote without being decoded, so the value is never decoded, joined or scanned as text.
+// A chunk longer than MAX_DECODE_SLICE is decoded in slices of that size, so the skip also applies
+// within one chunk: Node's native zstd emits whole frames that can hold a value entirely.
+// Figures: worker-protocol.md, "Decoder and parser calibration".
 const MAX_DECODE_SLICE = 512 * 1024;
 
 // A line joined from text decoded in more than one slice is a V8 cons string, which the first
@@ -438,10 +454,13 @@ export function createState()              {
     skippedLines: 0,
     unreadableSqlStarts: new Set(),
     accumState: new Map(),
+    executorAccumState: new Map(),
     rddInfo: new Map(),
     rddBlocks: new Map(),
     rddBlockUpdates: 0,
     taskAccumStages: new Map(),
+    lastTaskAccumIds: new Map(),
+    executorPeakMetrics: new Map(),
     pendingAdaptiveUpdates: new Map(),
     resolvedPlanExecutions: new Set(),
     evidenceInputs: {
@@ -480,36 +499,45 @@ export function normalizeSparkProperties(
 
 export { parseSparkMemoryMB };
 
-// Derive an allocated-resource summary from the Spark config map. Absent keys degrade to null,
-// not guessed defaults.
-export function extractResources(config                                           )                   {
+// Derive an allocated-resource summary from the Spark config map, each setting as the run's
+// effective conf has it (the logged value, else Spark's default for `sparkVersion`). A key with no
+// fixed default degrades to null, not a guess, and so does every key of a log that recorded no
+// Spark properties at all: there is nothing to tell a default from a missing config.
+export function extractResources(config                                           , sparkVersion                )                   {
   const cfg = config ?? {};
+  const recorded = Object.keys(cfg).length > 0;
+  const value = (k        )                     => (recorded ? effectiveSparkConf({ sparkVersion, properties: cfg }, k)?.value : undefined);
   const int = (k        ) => {
-    if (cfg[k] == null) return null;
-    const n = parseInt(cfg[k], 10);
+    const v = value(k);
+    if (v == null) return null;
+    const n = parseInt(v, 10);
     return Number.isFinite(n) ? n : null;
   };
-  const memMB = (k        ) => (cfg[k] != null ? parseSparkMemoryMB(cfg[k]) : null);
-  const bool = (k        ) => (cfg[k] != null ? String(cfg[k]).toLowerCase() === 'true' : null);
+  const memMB = (k        ) => parseSparkMemoryMB(value(k));
+  const bool = (k        ) => { const v = value(k); return v != null ? v.toLowerCase() === 'true' : null; };
   return {
     executor: {
+      // The executor size stays the logged one: Spark's 1g default does not apply to a local-mode run,
+      // which has no separate executor JVM, and the log does not say which master it ran under.
       memory: cfg['spark.executor.memory'] ?? null,
-      memoryMB: memMB('spark.executor.memory'),
-      memoryOverhead: cfg['spark.executor.memoryOverhead'] ?? null,
+      memoryMB: cfg['spark.executor.memory'] != null ? parseSparkMemoryMB(cfg['spark.executor.memory']) : null,
+      memoryOverhead: value('spark.executor.memoryOverhead') ?? null,
       memoryOverheadMB: memMB('spark.executor.memoryOverhead'),
       cores: int('spark.executor.cores'),
       instances: int('spark.executor.instances'),
     },
     driver: {
-      memory: cfg['spark.driver.memory'] ?? null,
+      memory: value('spark.driver.memory') ?? null,
       memoryMB: memMB('spark.driver.memory'),
-      memoryOverhead: cfg['spark.driver.memoryOverhead'] ?? null,
+      memoryOverhead: value('spark.driver.memoryOverhead') ?? null,
       memoryOverheadMB: memMB('spark.driver.memoryOverhead'),
       cores: int('spark.driver.cores'),
     },
     dynamicAllocationEnabled: bool('spark.dynamicAllocation.enabled'),
     shuffleServiceEnabled: bool('spark.shuffle.service.enabled'),
+    // The serializer a run set: Spark's default is reported as unset.
     serializer: cfg['spark.serializer'] ?? null,
+    stageExecutorMetricsLogging: bool('spark.eventLog.logStageExecutorMetrics'),
   };
 }
 
@@ -600,12 +628,45 @@ function taskRecordOf(event                                    , failure
     executorRunTime: m['Executor Run Time'] ?? 0,
     executorCpuTime: m['Executor CPU Time'] ?? 0,
     inputBytes: inp['Bytes Read'] ?? 0,
+    inputRecords: inp['Records Read'] ?? 0,
+    shuffleReadRecords: sr['Total Records Read'] ?? 0,
     outputBytes: out['Bytes Written'] ?? 0,
     outputRecords: out['Records Written'] ?? null,
   };
 }
 
-export function accumulateTask(event                                    , state             )       {
+// Max-folds the TaskEnd's executor metrics into the executor's run peak. Spark polls them at
+// executor heartbeat (spark.executor.metrics.pollingInterval defaults to 0), so a peak is a lower
+// bound. All-zero rows (local mode) carry no measurement and are skipped.
+function foldTaskExecutorMetrics(event                                    , state             )       {
+  const raw = event['Task Executor Metrics'];
+  const executorId = event['Task Info']?.['Executor ID'];
+  if (!raw || executorId == null) return;
+  let peaks                                    ;
+  let measured = false;
+  for (const [sparkName, ourName] of EXECUTOR_METRIC_FIELDS) {
+    const value = raw[sparkName];
+    if (typeof value !== 'number' || !(value > 0)) continue;
+    measured = true;
+    peaks ??= state.executorPeakMetrics.get(executorId) ?? {};
+    if (value > (peaks[ourName] ?? 0)) peaks[ourName] = value;
+  }
+  if (!measured || !peaks) return;
+  state.executorPeakMetrics.set(executorId, peaks);
+  if (executorId !== DRIVER_EXECUTOR_ID) state.evidenceInputs.executorMetricRows++;
+}
+
+function sameIds(a                   , b                               )          {
+  if (b === undefined || a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+  return true;
+}
+
+// `accumulableIds`, when given, are the event's accumulator IDs, which then are not in the event.
+export function accumulateTask(
+  event                                    , state             , accumulableIds                    ,
+)       {
+  foldTaskExecutorMetrics(event, state);
   const stageId = event['Stage ID'];
   const stage = state.stages.get(stageId);
   if (!stage) return null;
@@ -623,10 +684,13 @@ export function accumulateTask(event                                    , state
 
   state.evidenceInputs.taskRecords++;
 
-  const accumulables = event['Task Info']?.Accumulables ?? [];
-  for (const acc of accumulables) {
-    if (!state.taskAccumStages.has(acc.ID)) state.taskAccumStages.set(acc.ID, new Set());
-    state.taskAccumStages.get(acc.ID) .add(stageId);
+  const ids = accumulableIds ?? (event['Task Info']?.Accumulables ?? []).map((acc) => acc.ID);
+  if (!sameIds(ids, state.lastTaskAccumIds.get(stageId))) {
+    state.lastTaskAccumIds.set(stageId, ids);
+    for (const id of ids) {
+      if (!state.taskAccumStages.has(id)) state.taskAccumStages.set(id, new Set());
+      state.taskAccumStages.get(id) .add(stageId);
+    }
   }
 
   const info              = event['Task Info'] ?? {};
@@ -695,32 +759,40 @@ function accountLateSpeculativeLoser(event                                    ,
   return true;
 }
 
+                                                           
+
+// Driver-side values (DriverAccumUpdates) win over executor-side ones (StageCompleted
+// accumulables); a metric has one of the two sources, so the order only matters for a log that
+// carries both.
 export function resolvePlanTree(
   rootInfo               ,
   accumMap                     ,
   taskAccumStages                          ,
   executionStageIds                         ,
   executionId         ,
+  executorAccumMap                      ,
 )           {
   const seen = new WeakSet               ();
   const nodeMap = new WeakMap                         ();
   let nextId = 0;
-  // Prefixed with the owning SQL execution so PlanNode.id is globally unique,
-  // not just unique within this one call: two executions' trees can each
-  // produce a node named "n1", and a consumer keying off id alone (e.g. a
-  // findings-by-nodeId index) must be able to tell them apart without also
-  // threading execution scope through separately. No executionId (e.g. the
-  // legacy unit tests below) falls back to the old unprefixed "n0", "n1", ...
-  // shape.
+  // Prefixed with the owning SQL execution so PlanNode.id is globally unique: two executions'
+  // trees can each produce a node named "n1", and a consumer keying off id alone (e.g. a
+  // findings-by-nodeId index) must tell them apart. No executionId (hand-built trees in tests)
+  // leaves ids unprefixed: "n0", "n1", ...
   const idPrefix = executionId != null ? `e${executionId}:` : '';
 
   function computeMetricsAndStageIds(info               )   
-                                                                    
+                          
                         
     {
-    const metrics = (info.metrics ?? []).reduce                                                        ((acc, m) => {
-      if (m.accumulatorId !== undefined && accumMap.has(m.accumulatorId)) {
+    const metrics = (info.metrics ?? []).reduce              ((acc, m) => {
+      if (m.accumulatorId === undefined) return acc;
+      if (accumMap.has(m.accumulatorId)) {
         acc.push({ name: m.name, value: accumMap.get(m.accumulatorId) , metricType: m.metricType });
+      } else if (executorAccumMap?.has(m.accumulatorId) && m.metricType !== 'average') {
+        // An 'average' metric's accumulator holds a sum of per-task averages (scaled by 10), so
+        // its total is not the average the SQL tab shows; that needs the per-task values.
+        acc.push({ name: m.name, value: executorAccumMap.get(m.accumulatorId) , metricType: m.metricType, executorSide: true });
       }
       return acc;
     }, []);
@@ -826,17 +898,20 @@ const EXECUTOR_METRIC_FIELD_MAP                         = {
   MajorGCCount: 'majorGCCount', MajorGCTime: 'majorGCTime', TotalGCTime: 'totalGCTime',
   ConcurrentGCCount: 'concurrentGCCount', ConcurrentGCTime: 'concurrentGCTime',
 };
+const EXECUTOR_METRIC_FIELDS = Object.entries(EXECUTOR_METRIC_FIELD_MAP);
 
 export function startApplication(event                                             , state             ) {
   const config = state.pendingConfig ?? {};
   state.app = {
     id: event['App ID'],
     name: event['App Name'],
+    // Only in cluster-mode logs: Spark writes no attempt id for a client or local run.
+    ...(event['App Attempt ID'] ? { attemptId: event['App Attempt ID'] } : {}),
     startTime: event['Timestamp'],
     endTime: null,
     sparkVersion: state.pendingSparkVersion ?? event['Spark Version'] ?? null,
     config,
-    resources: state.pendingResources ?? extractResources(config),
+    resources: state.pendingResources ?? extractResources(config, state.pendingSparkVersion ?? event['Spark Version']),
     rddInfo: state.rddInfo,
   };
   return appMessage(state);
@@ -845,7 +920,7 @@ export function startApplication(event
 export function updateEnvironment(event                                              , state             ) {
   state.evidenceInputs.environmentUpdates++;
   const config = normalizeSparkProperties(event['Spark Properties']);
-  const resources = extractResources(config);
+  const resources = extractResources(config, state.app?.sparkVersion ?? state.pendingSparkVersion);
   // EnvironmentUpdate normally precedes ApplicationStart: stash the config so ApplicationStart can
   // attach it. If it arrives after (a mid-run update), apply live and re-post the app.
   if (state.app) {
@@ -1112,10 +1187,11 @@ export function recordStageExecutorMetrics(event
   if (!stage) return null;
   const raw = event['Executor Metrics'] ?? {};
   const metrics                         = {};
-  for (const [sparkName, ourName] of Object.entries(EXECUTOR_METRIC_FIELD_MAP)) {
+  for (const [sparkName, ourName] of EXECUTOR_METRIC_FIELDS) {
     if (raw[sparkName] != null) metrics[ourName] = raw[sparkName];
   }
-  if (Object.keys(metrics).length > 0) {
+  // A row for the driver's own heap is no executor measurement: the heap-peak finding leaves it out too.
+  if (Object.keys(metrics).length > 0 && event['Executor ID'] !== DRIVER_EXECUTOR_ID) {
     state.evidenceInputs.executorMetricRows++;
   }
   stage.executorMetrics.set(event['Executor ID'], metrics);
@@ -1132,6 +1208,7 @@ export function startSqlExecution(event
     hadAdaptiveUpdate: false,
   };
   if (event.rootExecutionId !== undefined) exec.rootExecutionId = event.rootExecutionId;
+  if (event.modifiedConfigs !== undefined && Object.keys(event.modifiedConfigs).length > 0) exec.modifiedConfigs = event.modifiedConfigs;
   if (event.physicalPlanDescription && KEPT_ARGUMENTS.test(event.physicalPlanDescription)) {
     exec.commandArguments = event.physicalPlanDescription;
   }
@@ -1140,6 +1217,7 @@ export function startSqlExecution(event
   state.resolvedPlanExecutions.delete(exec.id);
   if (sparkPlanInfo !== null) {
     state.accumState.set(exec.id, new Map());
+    state.executorAccumState.set(exec.id, new Map());
   }
   return { type: 'sql', data: exec };
 }
@@ -1157,14 +1235,17 @@ export function endSqlExecution(
   const planInfo = exec?.sparkPlanInfo ?? null;
   if (!planInfo || !planInfo.nodeName) {
     state.accumState.delete(event.executionId);
+    state.executorAccumState.delete(event.executionId);
     return exec ? { type: 'sql', data: { ...exec } } : null;
   }
 
   const accumMap = state.accumState.get(event.executionId) ?? new Map                ();
   const planTree = resolvePlanTree(
     planInfo, accumMap, state.taskAccumStages, state.sqlExecStages.get(event.executionId), event.executionId,
+    state.executorAccumState.get(event.executionId),
   );
   state.accumState.delete(event.executionId);
+  state.executorAccumState.delete(event.executionId);
   // The resolved tree is all anything downstream reads; the raw plan (often megabytes per
   // execution under AQE) would otherwise stay live for the rest of the parse.
   exec .sparkPlanInfo = null;
@@ -1174,6 +1255,24 @@ export function endSqlExecution(
   return { type: 'sqlPlan', data: { executionId: event.executionId, planTree } };
 }
 
+// Keeps the latest total of every SQL metric accumulator a stage reports, for the execution that
+// ran the stage. A stage's accumulables hold each accumulator's running total across all stages
+// and attempts, so a later stage overwrites an earlier one rather than adding to it (Spark's
+// DAGScheduler.updateAccumulators merges into the driver-side accumulator and records its value;
+// the SQL tab sums the same task updates). Spark writes the value as a decimal string.
+export function recordStageSqlAccumulables(info                                                         , state             )       {
+  const accumulables = info.Accumulables;
+  if (!accumulables || accumulables.length === 0) return;
+  const executionId = state.stageToSqlExec.get(info['Stage ID']);
+  const totals = executionId === undefined ? undefined : state.executorAccumState.get(executionId);
+  if (!totals) return;
+  for (const acc of accumulables) {
+    if (acc.Metadata !== 'sql') continue;
+    const value = typeof acc.Value === 'string' ? Number(acc.Value) : acc.Value;
+    if (typeof value === 'number' && Number.isFinite(value)) totals.set(acc.ID, value);
+  }
+}
+
 export function applyDriverAccumUpdates(event                                               , state             )       {
   const { executionId, accumUpdates } = event;
   if (!state.accumState.has(executionId)) return null; // late update, discard
@@ -1266,6 +1365,7 @@ export function processEvent(event            , state             )          {
       const id = info['Stage ID'];
       const stage = state.stages.get(id);
       if (!stage) return null;
+      recordStageSqlAccumulables(info, state);
       stage.completedAt = info['Completion Time'] ?? 0;
       // Older Spark (seen on 1.x-2.0 logs) posts StageSubmitted before the stage's submission
       // time is set; StageCompleted carries it. Without this backfill submittedAt stays 0 and
@@ -1321,9 +1421,8 @@ const KNOWN_EVENT_TYPES                      = new Set(
 );
 
 // SQLExecutionStart and SQLAdaptiveExecutionUpdate carry `physicalPlanDescription`, Spark's text
-// rendering of the plan. The plan tree comes from sparkPlanInfo, yet on a real 3.5 GB log the
-// description was 72% of the AQE-update bytes, which were themselves 73% of the log. Cutting its
-// string value out before JSON.parse halves the parse cost of those lines.
+// rendering of the plan. The plan tree comes from sparkPlanInfo, so the description's string value
+// is cut out before JSON.parse (worker-protocol.md, "Decoder and parser calibration").
 //
 // One exception: a Delta write command's node prints only its bare name in sparkPlanInfo, and its
 // target table appears only on its `Arguments:` line of the description. When the description is
@@ -1435,13 +1534,25 @@ const ACCUMULABLE_ID_KEY = '"ID":';
 const MAX_ACCUMULABLE_ID_DIGITS = 15;
 
 // Parses a TaskEnd line with its Task Info Accumulables array reduced to the `{ID}` entries
-// accumulateTask reads, or returns null for the caller to parse the line whole. That array is 71%
-// of TaskEnd bytes on the largest real log, where parsing its TaskEnd lines took 1.3s (0.5s here).
+// accumulateTask reads, or returns null for the caller to parse the line whole. That array is most
+// of a TaskEnd line's bytes (worker-protocol.md, "Decoder and parser calibration").
 // The IDs are read with a string scan and the array is cut out before JSON.parse, but only when
 // every entry has Spark's flat form (`{"ID":n,...}`, ID first, no nested array): any `{` that
 // doesn't open `{"ID":n` or any `[` in the array (updatedBlockStatuses) falls back. A `]` inside a
 // Name string cuts the array short and leaves invalid JSON, which falls back too.
 export function parseTaskEnd(line        )          {
+  const split = splitTaskEnd(line);
+  if (!split) return null;
+  split.parsed['Task Info'] .Accumulables = split.ids.map((ID) => ({ ID }));
+  return split.parsed;
+}
+
+                                                                                            
+
+// parseTaskEnd's work with the IDs left apart from the parsed event, whose Accumulables is empty:
+// the scan already read the IDs as digits, so parseAndDispatch hands them straight to accumulateTask
+// instead of validating them again.
+function splitTaskEnd(line        )                      {
   if (!line.startsWith(TASK_END_PREFIX)) return null;
   const keyAt = line.indexOf(ACCUMULABLES_KEY);
   if (keyAt === -1) return null;
@@ -1461,6 +1572,7 @@ export function parseTaskEnd(line        )          {
     if (after !== 0x2c && after !== 0x7d) return null; // `,` or `}`
     ids.push(id);
   }
+  if (ids.length > MAX_ACCUMULABLES_PER_TASK) return null; // the schema rejects it: parse whole
   let parsed                                                     ;
   try {
     parsed = JSON.parse(line.slice(0, from) + line.slice(close));
@@ -1469,8 +1581,7 @@ export function parseTaskEnd(line        )          {
   }
   const info = parsed?.['Task Info'];
   if (!info || !Array.isArray(info.Accumulables) || info.Accumulables.length !== 0) return null;
-  info.Accumulables = ids.map((ID) => ({ ID }));
-  return parsed;
+  return { parsed: parsed , ids };
 }
 
 const ADAPTIVE_UPDATE_PREFIX =
@@ -1481,16 +1592,14 @@ const MAX_EXECUTION_ID_DIGITS = 20;
 
 // An AQE update replaces its execution's whole plan, and nothing reads a plan an update replaced:
 // only the last one before SQLExecutionEnd is resolved. So while the execution is open, only its
-// latest update is kept, as unparsed text, and superseded ones are never parsed. On the largest
-// real log 541 of 664 updates were superseded, 1.1 MB of plan JSON each after the plan text is
-// cut. Returns false (parse it now, as any other line) unless the line is Spark's compact form
+// latest update is kept, as unparsed text, and superseded ones are never parsed (figures:
+// worker-protocol.md, "Decoder and parser calibration"). Returns false (parse it now, as any other line) unless the line is Spark's compact form
 // ending in an object-valued `sparkPlanInfo`, its last field: a null-plan update keeps the
 // previous plan, so it can't supersede one. Trade-off: a malformed superseded update is never
-// seen, so it no longer counts as a skipped line.
+// seen, so it does not count as a skipped line.
 //
 // These updates span decode slices, so `line` is a cons string: the prefix and suffix checks read
-// `joined`'s flat pieces instead, or a superseded update is copied flat only to be dropped (541
-// copies of 1.1 MB on that log). A piece too short to hold what is checked falls back to `line`.
+// `joined`'s flat pieces instead, or a superseded update is copied flat only to be dropped. A piece too short to hold what is checked falls back to `line`.
 function deferAdaptiveUpdate(
   line        , state             , emit                        , joined             ,
 )          {
@@ -1548,8 +1657,11 @@ function noteUnreadableSqlStart(line        , state             )       {
 function parseAndDispatch(line        , state             , emit                        )       {
   if (line.startsWith(BLOCK_UPDATED_PREFIX) && !line.includes(RDD_BLOCK_ID_FRAGMENT)) return;
   let parsed         ;
+  let taskEndIds                      ;
   try {
-    parsed = parseTaskEnd(line) ?? JSON.parse(stripPlanDescription(line));
+    const split = splitTaskEnd(line);
+    if (split) ({ parsed, ids: taskEndIds } = split);
+    else parsed = JSON.parse(stripPlanDescription(line));
   } catch {
     state.skippedLines++;
     noteUnreadableSqlStart(line, state);
@@ -1573,7 +1685,9 @@ function parseAndDispatch(line        , state             , emit
   }
   let msg         ;
   try {
-    msg = processEvent(result.data, state);
+    msg = taskEndIds && result.data.Event === 'SparkListenerTaskEnd'
+      ? accumulateTask(result.data, state, taskEndIds)
+      : processEvent(result.data, state);
   } catch (err) {
     // processEvent's switch is exhaustive over the schema-validated union (assertNever default), so
     // this is unreachable for any event that got this far. If it fires it's a handler bug, not
@@ -1623,11 +1737,14 @@ export function collectLateAttemptWork(state             )
 }
 
 export function emitParseCompletion(state             , emit                        , linesProcessed        )       {
-  // Executions that never ended keep their latest AQE update, as they did before it was deferred.
+  // Executions that never ended still apply their latest deferred AQE update.
   for (const executionId of [...state.pendingAdaptiveUpdates.keys()]) flushAdaptiveUpdate(executionId, state, emit);
   emit({ type: 'progress', pct: 1, linesProcessed });
   emit({ type: 'stageLateAttemptWork', data: collectLateAttemptWork(state) });
-  emit({ type: 'runAggregates', data: computeRunAggregates(state.taskStore) });
+  emit({
+    type: 'runAggregates',
+    data: { ...computeRunAggregates(state.taskStore), executorPeakMetrics: Object.fromEntries(state.executorPeakMetrics) },
+  });
   emit({ type: 'stageSpeculationWaste', data: collectLateSpeculationWaste(state) });
   emit({ type: 'stageExecutorMetrics', data: collectStageExecutorMetrics(state) });
   emit(appMessage(state));
@@ -1637,4 +1754,5 @@ export function emitParseCompletion(state             , emit
     ...(state.unreadableSqlStarts.size > 0 ? { unreadableSqlExecutions: [...state.unreadableSqlStarts].sort((a, b) => a - b) } : {}),
   });
   state.accumState.clear();
+  state.executorAccumState.clear();
 }