sparkforensics-mcp 0.3.0 → 0.5.0

package/vendor-core/event-handlers.js

@@ -14,6 +14,7 @@ import {
   DriverAccumUpdatesEventSchema,
   ExecutorAddedEventSchema,
   ExecutorRemovedEventSchema,
+  BlockUpdatedEventSchema,
                   
                      
 } from './event-schemas.js';
@@ -21,8 +22,9 @@ import { assertNever } from './assert-never.js';
 import { finalizeStage } from './stage-quantiles.js';
 import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
 import { computeRunAggregates } from './run-aggregates.js';
+import { parseSparkMemoryMB } from './spark-memory.js';
              
-                                                                                        
+                                                                                                            
                  
 
 // Internal parser-state shapes: the real runtime objects the handlers build and mutate, not the
@@ -62,7 +64,22 @@ import { computeRunAggregates } from './run-aggregates.js';
                               
                      
                    
+                                                                                                    
+                                                                                                          
+                                                                                              
+                                            
                         
+ 
+
+// Live per-RDD block residency rebuilt from SparkListenerBlockUpdated. Keyed by partition and
+// executor because a block's status is per BlockManager: replicas and re-caches on another
+// executor are separate entries, as in Spark's own AppStatusListener.
+                         
+                                                                                   
+                                           
+                     
+                   
+                               
  
 
                                                                                       
@@ -113,6 +130,8 @@ const MAX_TASK_SAMPLES = 20;
                           
                      
                       
+                                                                                                
+                               
  
 
                        
@@ -133,12 +152,29 @@ const MAX_TASK_SAMPLES = 20;
                           
                      
                       
+                                                                      
+                               
                                 
                       
                                   
                            
                                       
                                     
+                                                                                                        
+                         
+                                                                                                  
+                                                                                       
+                        
+                              
+                                                                                       
+                                             
+                                                                                       
+                                                                                             
+                                                                                              
+                                                     
+                                             
+                                                              
+                                  
                                                         
                                                                                            
                                                         
@@ -147,6 +183,12 @@ const MAX_TASK_SAMPLES = 20;
                          
                              
                                     
+                                                                                                
+                                                                                                          
+                                           
+                                                                                             
+                                               
+                                
                                                        
  
 
@@ -176,8 +218,15 @@ const MAX_TASK_SAMPLES = 20;
                          
                                                                               
                        
+                                                                                               
+                                                                                  
+                                   
                                                
                                       
+                                        
+                                                                                                 
+                                                                      
+                          
                                             
                                  
                                                                                      
@@ -365,8 +414,11 @@ export function createState()              {
     jobs: new Map(),
     executors: { added: [], removed: [] },
     skippedLines: 0,
+    unreadableSqlStarts: new Set(),
     accumState: new Map(),
     rddInfo: new Map(),
+    rddBlocks: new Map(),
+    rddBlockUpdates: 0,
     taskAccumStages: new Map(),
     pendingAdaptiveUpdates: new Map(),
     resolvedPlanExecutions: new Set(),
@@ -404,23 +456,7 @@ export function normalizeSparkProperties(
   return map;
 }
 
-// Parse a Spark memory-size string to MiB. Spark's JVM-memory configs use bytesConf(ByteUnit.MiB),
-// so a bare number means MiB. A k/m/g/t suffix sets the unit (trailing "b" redundant); a lone "b"
-// ("10b") means bytes.
-export function parseSparkMemoryMB(value         )                {
-  if (value == null) return null;
-  const m = String(value).trim().toLowerCase().match(/^([\d.]+)\s*([kmgt]?)(b?)$/);
-  if (!m) return null;
-  const n = parseFloat(m[1]);
-  if (!Number.isFinite(n)) return null;
-  switch (m[2]) {
-    case 'k': return Math.round(n / 1024);
-    case 'g': return Math.round(n * 1024);
-    case 't': return Math.round(n * 1024 * 1024);
-    case 'm': return Math.round(n);
-    default: return m[3] === 'b' ? Math.round(n / (1024 * 1024)) : Math.round(n);
-  }
-}
+export { parseSparkMemoryMB };
 
 // Derive an allocated-resource summary from the Spark config map. Absent keys degrade to null,
 // not guessed defaults.
@@ -478,6 +514,7 @@ function snapshotEvidenceInputs(state             )                 {
 function appMessage(state             )                                                 {
   const evidenceInputs = snapshotEvidenceInputs(state);
   state.app .evidenceInputs = evidenceInputs;
+  state.app .rddBlockUpdates = state.rddBlockUpdates;
   return {
     type: 'app',
     data: { ...state.app , rddInfo: snapshotRddInfo(state.rddInfo) },
@@ -504,25 +541,11 @@ function internTaskFailure(stage             , endReason
   return detail;
 }
 
-export function accumulateTask(event                                    , state             )       {
-  const stageId = event['Stage ID'];
-  const stage = state.stages.get(stageId);
-  if (!stage) return null;
-  // Late TaskEnd for a stage whose StageCompleted already freed taskAttempts (finalizeStage): its
-  // stats are already baked into the finalized stage, don't re-add.
-  if (stage.taskAttempts === null) return null;
-
-  state.evidenceInputs.taskRecords++;
-
-  const accumulables = event['Task Info']?.Accumulables ?? [];
-  for (const acc of accumulables) {
-    if (!state.taskAccumStages.has(acc.ID)) state.taskAccumStages.set(acc.ID, new Set());
-    state.taskAccumStages.get(acc.ID) .add(stageId);
-  }
+// 'Task Info' and its Failed/Killed/Speculative fields are optional in the schema; Partial<>
+// lets the {} fallback type-check while reads below default via ??/||.
+                                                                                         
 
-  // 'Task Info' and its Failed/Killed/Speculative fields are optional in the schema; Partial<>
-  // lets the {} fallback type-check while reads below default via ??/||.
-                                                                                           
+function taskRecordOf(event                                    , failure                          )             {
   const info              = event['Task Info'] ?? {};
   const m = event['Task Metrics'] ?? {};
   const sr = m['Shuffle Read Metrics'] ?? {};
@@ -533,14 +556,14 @@ export function accumulateTask(event                                    , state
   const duration = (info['Finish Time'] ?? 0) - (info['Launch Time'] ?? 0);
   const failed = !!(info['Failed'] || info['Killed']);
 
-  const record             = {
+  return {
     duration, failed,
     taskId: info['Task ID'] ?? null,
     attemptNumber: info['Attempt Number'] ?? 0,
     launchTime: info['Launch Time'] ?? 0,
     finishTime: info['Finish Time'] ?? 0,
     reason: event['Task End Reason']?.['Reason'] ?? null,
-    failure: failed ? internTaskFailure(stage, event['Task End Reason']) : null,
+    failure,
     speculative: info['Speculative'] === true,
     host: info['Host'] ?? '',
     executorId: info['Executor ID'] ?? '',
@@ -556,7 +579,37 @@ export function accumulateTask(event                                    , state
     executorCpuTime: m['Executor CPU Time'] ?? 0,
     inputBytes: inp['Bytes Read'] ?? 0,
     outputBytes: out['Bytes Written'] ?? 0,
+    outputRecords: out['Records Written'] ?? null,
   };
+}
+
+export function accumulateTask(event                                    , state             )       {
+  const stageId = event['Stage ID'];
+  const stage = state.stages.get(stageId);
+  if (!stage) return null;
+  // Late TaskEnd for a stage whose StageCompleted already freed taskAttempts (finalizeStage): the
+  // finalized stage's figures stay as posted. A losing speculative attempt adds the wasted time the
+  // finalized stage never saw; any other task of a failed or earlier attempt is work only the
+  // metrics block reads, from lateAttemptWork.
+  if (stage.taskAttempts === null) {
+    const earlierAttempt = (event['Stage Attempt ID'] ?? 0) !== stage.stageAttemptId;
+    if (!accountLateSpeculativeLoser(event, stage) && (stage.stageFailureReason != null || earlierAttempt)) {
+      stage.lateAttemptWork = mergeAttemptTotals(taskAttemptTotals(taskRecordOf(event, null)), stage.lateAttemptWork);
+    }
+    return null;
+  }
+
+  state.evidenceInputs.taskRecords++;
+
+  const accumulables = event['Task Info']?.Accumulables ?? [];
+  for (const acc of accumulables) {
+    if (!state.taskAccumStages.has(acc.ID)) state.taskAccumStages.set(acc.ID, new Set());
+    state.taskAccumStages.get(acc.ID) .add(stageId);
+  }
+
+  const info              = event['Task Info'] ?? {};
+  const failed = !!(info['Failed'] || info['Killed']);
+  const record = taskRecordOf(event, failed ? internTaskFailure(stage, event['Task End Reason']) : null);
 
   // Dedupe only when Index is present (always true for real logs). Without it every event is a
   // distinct task, preserving behavior for fixtures that omit Index.
@@ -566,6 +619,7 @@ export function accumulateTask(event                                    , state
 
   if (!existing) {
     stage.taskAttempts.set(key, record);
+    if (record.speculative && !record.failed) stage.speculativeWinners.add(key);
   } else if (existing.failed && !record.failed) {
     // A retry succeeded where the earlier attempt failed: the earlier attempt's time was wasted.
     // Spark marks only the speculative COPY's Speculative flag, never the original it raced, so
@@ -580,8 +634,11 @@ export function accumulateTask(event                                    , state
         stage.retryTaskSamples.push(taskRecordToSample(existing));
       }
     }
+    stage.lateAttemptWork = mergeAttemptTotals(discardedAttemptTotals(existing), stage.lateAttemptWork);
     stage.taskAttempts.set(key, record);
+    if (record.speculative) stage.speculativeWinners.add(key);
   } else {
+    stage.lateAttemptWork = mergeAttemptTotals(discardedAttemptTotals(record), stage.lateAttemptWork);
     // Non-winning duplicate (both failed, or a race where a winner is
     // already recorded): its time is waste, its metrics are discarded.
     if (existing.speculative || record.speculative) {
@@ -599,6 +656,23 @@ export function accumulateTask(event                                    , state
   return null;
 }
 
+// Spark kills the losing copy of a speculative race only once the stage finishes ("Stage
+// cancelled: Stage finished"), so that loser's TaskEnd normally lands after StageCompleted. Count
+// its time as speculation waste, pairing it the same way accumulateTask does: the late attempt is
+// the speculative copy itself, or the original that a speculative winner beat. Every other stat
+// of a late attempt stays excluded, as the finalized stage already posted them.
+function accountLateSpeculativeLoser(event                                    , stage             )          {
+  const info = event['Task Info'];
+  if (info?.['Index'] == null) return false;
+  const key = `${event['Stage Attempt ID'] ?? 0}:${info['Index']}`;
+  if (info['Speculative'] !== true && !stage.speculativeWinners.has(key)) return false;
+  stage.speculationWasteMs += (info['Finish Time'] ?? 0) - (info['Launch Time'] ?? 0);
+  stage.speculationWastedAttempts++;
+  stage.lateSpeculationWaste = true;
+  stage.lateAttemptWork = mergeAttemptTotals(discardedAttemptTotals(taskRecordOf(event, null)), stage.lateAttemptWork);
+  return true;
+}
+
 export function resolvePlanTree(
   rootInfo               ,
   accumMap                     ,
@@ -803,10 +877,63 @@ export function endJob(event                                   , state
   return { type: 'job', data: { ...job } };
 }
 
+const SUMMED_ATTEMPT_FIELDS = [
+  'taskCount', 'failedTasks', 'wastedAttempts', 'executorRunTime', 'executorCpuTime', 'jvmGCTime',
+  'memoryBytesSpilled', 'diskBytesSpilled', 'shuffleReadBytes', 'shuffleWriteBytes', 'inputBytes', 'outputBytes',
+]         ;
+
+const addNullable = (a               , b               )                => (a == null ? b : b == null ? a : a + b);
+
+function mergeAttemptTotals(a                    , b                           )                     {
+  if (b == null) return a;
+  const totals = {
+    outputRecords: addNullable(a.outputRecords, b.outputRecords),
+    peakExecutionMemoryMax: Math.max(a.peakExecutionMemoryMax, b.peakExecutionMemoryMax),
+    durationMs: addNullable(a.durationMs, b.durationMs),
+  }                      ;
+  for (const field of SUMMED_ATTEMPT_FIELDS) totals[field] = a[field] + b[field];
+  return totals;
+}
+
+// One task's work; a late task adds no stage duration.
+function taskAttemptTotals(t            )                     {
+  return {
+    taskCount: 1, failedTasks: t.failed ? 1 : 0, wastedAttempts: 0,
+    executorRunTime: t.executorRunTime, executorCpuTime: t.executorCpuTime, jvmGCTime: t.gcTime,
+    memoryBytesSpilled: t.memSpilled, diskBytesSpilled: t.diskSpilled,
+    shuffleReadBytes: t.shuffleRead, shuffleWriteBytes: t.shuffleWrite,
+    inputBytes: t.inputBytes, outputBytes: t.outputBytes, outputRecords: t.outputRecords,
+    peakExecutionMemoryMax: t.peakExecMem, durationMs: null,
+  };
+}
+
+// A task attempt the stage's figures drop because another attempt of the same task won (a retry
+// after a failure, a speculative twin): its CPU, run time and I/O were still spent, so the
+// metrics block counts them, but the task itself is already counted once.
+function discardedAttemptTotals(t            )                     {
+  return { ...taskAttemptTotals(t), taskCount: 0, failedTasks: 0 };
+}
+
+// The replaced record's finalized attempt added to the attempts it had already folded. An attempt resubmitted before its StageCompleted was never finalized, so its
+// tasks are not counted.
+function foldEarlierAttempts(replaced                         )                            {
+  if (!replaced) return null;
+  if (replaced.taskAttempts !== null) return replaced.earlierAttempts;
+  const attempt                     = {
+    ...Object.fromEntries(SUMMED_ATTEMPT_FIELDS.map((field) => [field, replaced[field]]))                                                                  ,
+    outputRecords: replaced.outputRecords,
+    peakExecutionMemoryMax: replaced.peakExecutionMemoryMax ?? 0,
+    durationMs: replaced.submittedAt > 0 && replaced.completedAt >= replaced.submittedAt
+      ? replaced.completedAt - replaced.submittedAt : null,
+  };
+  return mergeAttemptTotals(attempt, replaced.earlierAttempts);
+}
+
 export function submitStage(event                                           , state             )       {
   state.evidenceInputs.stageSubmissions++;
   const info = event['Stage Info'];
   const id = info['Stage ID'];
+  const replaced = state.stages.get(id);
   state.stages.set(id, {
     id, name: info['Stage Name'] ?? '', details: info['Details'] ?? '',
     submittedAt: info['Submission Time'] ?? 0, completedAt: 0,
@@ -814,13 +941,18 @@ export function submitStage(event                                           , st
     shuffleReadBytes: 0, shuffleWriteBytes: 0, fetchWaitTime: 0,
     memoryBytesSpilled: 0, diskBytesSpilled: 0,
     jvmGCTime: 0, executorRunTime: 0, executorCpuTime: 0,
-    inputBytes: 0, outputBytes: 0,
+    inputBytes: 0, outputBytes: 0, outputRecords: null,
     sqlExecutionId: state.stageToSqlExec.get(id) ?? null,
     parentIds: info['Parent IDs'] ?? [],
     hostStats: new Map(),
     speculativeTasks: 0,
     failureReasons: new Map(),
     stageFailureReason: null,
+    stageAttemptId: info['Stage Attempt ID'] ?? 0,
+    stageAttempts: (replaced?.stageAttempts ?? 0) + 1,
+    failedStageAttempts: replaced?.failedStageAttempts ?? 0,
+    earlierAttempts: foldEarlierAttempts(replaced),
+    lateAttemptWork: replaced?.lateAttemptWork ?? null,
     taskAttempts: new Map(),
     failureDetails: new Map(),
     retryTaskSamples: [],
@@ -828,6 +960,8 @@ export function submitStage(event                                           , st
     wastedAttempts: 0,
     speculationWasteMs: 0,
     speculationWastedAttempts: 0,
+    speculativeWinners: new Set(),
+    lateSpeculationWaste: false,
     executorMetrics: new Map(),
   });
   mergeStageRddInfo(info, id, state);
@@ -847,11 +981,16 @@ export function mergeStageRddInfo(
     const prev = state.rddInfo.get(rddId);
     const stageIds = prev?.stageIds ?? new Set        ();
     stageIds.add(id);
+    // Block updates are the authoritative source once seen: never let a later RDD Info snapshot
+    // (0 on Spark 2.3+) replace them, nor the NONE level an unpersist() leaves on ancestor RDDs
+    // listed by later stages.
+    const fromBlocks = prev?.storageSource === 'blockUpdates';
+    const persisted = Boolean(sl['Use Disk'] || sl['Use Memory']);
     state.rddInfo.set(rddId, {
       id: rddId,
       name: rdd['Name'] ?? '',
       callsite: rdd['Callsite'] ?? '',
-      storageLevel: {
+      storageLevel: fromBlocks && !persisted ? prev.storageLevel : {
         useDisk: sl['Use Disk'] ?? false,
         useMemory: sl['Use Memory'] ?? false,
         deserialized: sl['Deserialized'] ?? false,
@@ -861,14 +1000,88 @@ export function mergeStageRddInfo(
       // Merge forward, don't overwrite: an RDD cached for the first time in THIS stage legitimately
       // reports 0 (snapshot reflects BlockManager state at submission). Keep the last real value on
       // a resubmission instead of regressing to 0.
-      numCachedPartitions: rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
-      memorySize: rdd['Memory Size'] || prev?.memorySize || 0,
-      diskSize: rdd['Disk Size'] || prev?.diskSize || 0,
+      numCachedPartitions: fromBlocks ? prev.numCachedPartitions : rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
+      memorySize: fromBlocks ? prev.memorySize : rdd['Memory Size'] || prev?.memorySize || 0,
+      diskSize: fromBlocks ? prev.diskSize : rdd['Disk Size'] || prev?.diskSize || 0,
+      storageSource: prev?.storageSource ?? 'rddInfo',
       stageIds,
     });
   }
 }
 
+const RDD_BLOCK_ID = /^rdd_(\d+)_(\d+)$/;
+
+function dropRddBlock(rdd               , key        )       {
+  const prev = rdd.blocks.get(key);
+  if (!prev) return;
+  rdd.memorySize -= prev.memorySize;
+  rdd.diskSize -= prev.diskSize;
+  const replicas = (rdd.replicasByPartition.get(prev.partition) ?? 1) - 1;
+  if (replicas > 0) rdd.replicasByPartition.set(prev.partition, replicas);
+  else rdd.replicasByPartition.delete(prev.partition);
+  rdd.blocks.delete(key);
+}
+
+/**
+ * Folds one SparkListenerBlockUpdated into its RDD's live residency, then publishes the RDD's
+ * peak state to rddInfo: the most partitions resident at once, with the memory/disk bytes at the
+ * latest moment that peak held. A peak rather than the final state, because an unpersist() (or
+ * the app's own cleanup) removes every block before the log ends, and a final snapshot would
+ * read as "nothing was cached". Ties refresh, so a partition dropping from memory to disk after
+ * the peak still shows up in diskSize. Non-RDD blocks (broadcast, shuffle, task results) are ignored.
+ */
+export function recordBlockUpdate(event                                         , state             )       {
+  const info = event['Block Updated Info'];
+  const match = RDD_BLOCK_ID.exec(info['Block ID']);
+  if (!match) return null;
+  state.rddBlockUpdates++;
+  const rddId = Number(match[1]);
+  const partition = Number(match[2]);
+  const sl = info['Storage Level'] ?? {};
+  // Spark's StorageLevel.isValid: a removal or eviction reports level NONE.
+  const resident = Boolean(sl['Use Memory'] || sl['Use Disk']) && (sl['Replication'] ?? 1) > 0;
+
+  let rdd = state.rddBlocks.get(rddId);
+  if (!rdd) {
+    rdd = { blocks: new Map(), replicasByPartition: new Map(), memorySize: 0, diskSize: 0, peakCachedPartitions: 0 };
+    state.rddBlocks.set(rddId, rdd);
+  }
+  const key = `${partition}@${info['Block Manager ID']?.['Executor ID'] ?? ''}`;
+  dropRddBlock(rdd, key);
+  if (resident) {
+    // Sizes count only where the level says the block lives, as Spark's AppStatusListener does: a
+    // drop from memory to disk reports Use Memory false but still carries the dropped bytes as
+    // Memory Size (BlockManager reports max(memSize, droppedMemorySize)).
+    const block = {
+      partition,
+      memorySize: sl['Use Memory'] ? info['Memory Size'] ?? 0 : 0,
+      diskSize: sl['Use Disk'] ? info['Disk Size'] ?? 0 : 0,
+    };
+    rdd.blocks.set(key, block);
+    rdd.memorySize += block.memorySize;
+    rdd.diskSize += block.diskSize;
+    rdd.replicasByPartition.set(partition, (rdd.replicasByPartition.get(partition) ?? 0) + 1);
+  }
+
+  const record = state.rddInfo.get(rddId) ?? {
+    // A block reported before any stage listed its RDD: name and partition count arrive with
+    // the next StageSubmitted (mergeStageRddInfo keeps the block-derived sizes).
+    id: rddId, name: '', callsite: '',
+    storageLevel: { useDisk: Boolean(sl['Use Disk']), useMemory: Boolean(sl['Use Memory']), deserialized: false, replication: sl['Replication'] ?? 1 },
+    numPartitions: 0, numCachedPartitions: 0, memorySize: 0, diskSize: 0,
+    storageSource: 'blockUpdates'         , stageIds: new Set        (),
+  };
+  record.storageSource = 'blockUpdates';
+  if (rdd.replicasByPartition.size >= rdd.peakCachedPartitions) {
+    rdd.peakCachedPartitions = rdd.replicasByPartition.size;
+    record.numCachedPartitions = rdd.peakCachedPartitions;
+    record.memorySize = rdd.memorySize;
+    record.diskSize = rdd.diskSize;
+  }
+  state.rddInfo.set(rddId, record);
+  return null;
+}
+
 // Spark logs these AFTER SparkListenerStageCompleted, so finalizeStage has already posted the
 // stage with an empty executorMetrics Map. Keep accumulating worker-side, then re-post all maps
 // once via `stageExecutorMetrics` just before `done` so main-thread stages are patched before analyze().
@@ -982,6 +1195,14 @@ export function removeExecutor(event
     reason: event['Removed Reason'] ?? '',
   };
   state.executors.removed.push(ev);
+  // Blocks lost with their executor get no BlockUpdated; drop them as Spark's AppStatusListener
+  // does, so a partition re-cached elsewhere isn't counted twice.
+  const suffix = `@${ev.executorId}`;
+  for (const rdd of state.rddBlocks.values()) {
+    for (const key of rdd.blocks.keys()) {
+      if (key.endsWith(suffix)) dropRddBlock(rdd, key);
+    }
+  }
   return { type: 'executor', data: ev };
 }
 
@@ -1025,6 +1246,8 @@ export function processEvent(event            , state             )          {
       // every stage-duration figure becomes the epoch timestamp itself (a "47-year" stage).
       if (!stage.submittedAt && info['Submission Time'] != null) stage.submittedAt = info['Submission Time'];
       stage.stageFailureReason = info['Failure Reason'] ?? null;
+      // A duplicate StageCompleted of an already-finalized attempt is not another failed attempt.
+      if (stage.stageFailureReason != null && stage.taskAttempts !== null) stage.failedStageAttempts++;
       // finalizeStage keeps its `stage` parameter typed as a loose Record (see that module); bridge
       // StageRecord's more precise shape across that boundary with an explicit cast.
       return finalizeStage(
@@ -1058,6 +1281,9 @@ export function processEvent(event            , state             )          {
     case 'SparkListenerExecutorRemoved':
       return removeExecutor(event, state);
 
+    case 'SparkListenerBlockUpdated':
+      return recordBlockUpdate(event, state);
+
     default:
       return assertNever(event);
   }
@@ -1215,12 +1441,32 @@ export function dispatchLine(
   parseAndDispatch(line, state, emit);
 }
 
+// With logBlockUpdates on, most BlockUpdated lines are broadcast/shuffle blocks recordBlockUpdate
+// ignores. Spark writes "Event" first and "Block ID" as a plain string, so those lines can be
+// dropped on a substring test before paying for JSON.parse.
+const BLOCK_UPDATED_PREFIX = '{"Event":"SparkListenerBlockUpdated",';
+const RDD_BLOCK_ID_FRAGMENT = '"Block ID":"rdd_';
+
+const SQL_START_EVENT = 'org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart';
+const SQL_START_ID = /"executionId":(\d+)/;
+
+// Records the execution id of a skipped SQL start line, when the line's head still names it: a
+// line cut off mid-plan fails JSON.parse but keeps its leading fields.
+function noteUnreadableSqlStart(line        , state             )       {
+  const head = line.slice(0, 300);
+  if (!head.includes(`${SQL_START_EVENT}"`)) return;
+  const id = SQL_START_ID.exec(head);
+  if (id) state.unreadableSqlStarts.add(Number(id[1]));
+}
+
 function parseAndDispatch(line        , state             , emit                        )       {
+  if (line.startsWith(BLOCK_UPDATED_PREFIX) && !line.includes(RDD_BLOCK_ID_FRAGMENT)) return;
   let parsed         ;
   try {
     parsed = parseTaskEnd(line) ?? JSON.parse(stripPlanDescription(line));
   } catch {
     state.skippedLines++;
+    noteUnreadableSqlStart(line, state);
     return;
   }
   // A real Spark event log carries many event types this tool never modeled (BlockManagerAdded,
@@ -1233,6 +1479,7 @@ function parseAndDispatch(line        , state             , emit
   const result = SparkEventSchema.safeParse(parsed);
   if (!result.success) {
     state.skippedLines++;
+    if (eventType === SQL_START_EVENT) noteUnreadableSqlStart(line, state);
     return;
   }
   if (result.data.Event === 'org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd') {
@@ -1265,13 +1512,43 @@ export function collectStageExecutorMetrics(state             )
   return out;
 }
 
+// Speculation totals of every stage a late TaskEnd added waste to (accountLateSpeculativeLoser),
+// re-posted once before `done`: the stage message posted at completion carried the earlier totals.
+export function collectLateSpeculationWaste(
+  state             ,
+)                                                                                 {
+  const out = new Map                                                                           ();
+  for (const [id, stage] of state.stages) {
+    if (stage.lateSpeculationWaste) {
+      out.set(id, { speculationWasteMs: stage.speculationWasteMs, speculationWastedAttempts: stage.speculationWastedAttempts });
+    }
+  }
+  return out;
+}
+
+// Late work of every stage a failed attempt's late TaskEnd added to, re-posted once before `done`:
+// the stage message posted at completion predates it.
+export function collectLateAttemptWork(state             )                                  {
+  const out = new Map                            ();
+  for (const [id, stage] of state.stages) {
+    if (stage.lateAttemptWork != null) out.set(id, stage.lateAttemptWork);
+  }
+  return out;
+}
+
 export function emitParseCompletion(state             , emit                        , linesProcessed        )       {
   // Executions that never ended keep their latest AQE update, as they did before it was deferred.
   for (const executionId of [...state.pendingAdaptiveUpdates.keys()]) flushAdaptiveUpdate(executionId, state, emit);
   emit({ type: 'progress', pct: 1, linesProcessed });
+  emit({ type: 'stageLateAttemptWork', data: collectLateAttemptWork(state) });
   emit({ type: 'runAggregates', data: computeRunAggregates(state.taskStore) });
+  emit({ type: 'stageSpeculationWaste', data: collectLateSpeculationWaste(state) });
   emit({ type: 'stageExecutorMetrics', data: collectStageExecutorMetrics(state) });
   emit(appMessage(state));
-  emit({ type: 'done', skippedLines: state.skippedLines });
+  emit({
+    type: 'done',
+    skippedLines: state.skippedLines,
+    ...(state.unreadableSqlStarts.size > 0 ? { unreadableSqlExecutions: [...state.unreadableSqlStarts].sort((a, b) => a - b) } : {}),
+  });
   state.accumState.clear();
 }

package/vendor-core/event-schemas.js

@@ -201,6 +201,7 @@ export const StageSubmittedEventSchema = z.object({
   Event: z.literal('SparkListenerStageSubmitted'),
   'Stage Info': z.object({
     'Stage ID': z.number(),
+    'Stage Attempt ID': z.number().optional(),
     'Stage Name': z.string().optional(),
     Details: z.string().optional(),
     'Submission Time': z.number().optional(),
@@ -209,6 +210,26 @@ export const StageSubmittedEventSchema = z.object({
   }),
 });
 
+// recordBlockUpdate. Written only with spark.eventLog.logBlockUpdates.enabled=true; one event per
+// block status change reported to the driver's BlockManagerMaster (a removal or eviction carries
+// storage level NONE with zero sizes). Only 'rdd_<rddId>_<partition>' blocks are read.
+export const BlockUpdatedEventSchema = z.object({
+  Event: z.literal('SparkListenerBlockUpdated'),
+  'Block Updated Info': z.object({
+    'Block Manager ID': z.object({
+      'Executor ID': z.string().optional(),
+    }).optional(),
+    'Block ID': z.string(),
+    'Storage Level': z.object({
+      'Use Disk': z.boolean().optional(),
+      'Use Memory': z.boolean().optional(),
+      Replication: z.number().optional(),
+    }).optional(),
+    'Memory Size': z.number().optional(),
+    'Disk Size': z.number().optional(),
+  }),
+});
+
 // inline StageCompleted case.
 export const StageCompletedEventSchema = z.object({
   Event: z.literal('SparkListenerStageCompleted'),
@@ -291,6 +312,7 @@ export const TaskEndEventSchema = z.object({
     }).optional(),
     'Output Metrics': z.object({
       'Bytes Written': z.number().optional(),
+      'Records Written': z.number().optional(),
     }).optional(),
   }).optional(),
 });
@@ -345,5 +367,6 @@ export const SparkEventSchema = z.discriminatedUnion('Event', [
   DriverAccumUpdatesEventSchema,
   ExecutorAddedEventSchema,
   ExecutorRemovedEventSchema,
+  BlockUpdatedEventSchema,
 ]);