sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/event-handlers.js

@@ -14,11 +14,13 @@ import {
   DriverAccumUpdatesEventSchema,
   ExecutorAddedEventSchema,
   ExecutorRemovedEventSchema,
+  BlockUpdatedEventSchema,
                   
                      
 } from './event-schemas.js';
 import { assertNever } from './assert-never.js';
 import { finalizeStage } from './stage-quantiles.js';
+import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
 import { computeRunAggregates } from './run-aggregates.js';
              
                                                                                         
@@ -61,7 +63,22 @@ import { computeRunAggregates } from './run-aggregates.js';
                               
                      
                    
+                                                                                                    
+                                                                                                          
+                                                                                              
+                                            
                         
+ 
+
+// Live per-RDD block residency rebuilt from SparkListenerBlockUpdated. Keyed by partition and
+// executor because a block's status is per BlockManager: replicas and re-caches on another
+// executor are separate entries, as in Spark's own AppStatusListener.
+                         
+                                                                                   
+                                           
+                     
+                   
+                               
  
 
                                                                                       
@@ -94,6 +111,9 @@ const MAX_TASK_SAMPLES = 20;
                      
                      
                         
+                                                                                        
+                                                                      
+                                    
                        
                
                      
@@ -136,11 +156,19 @@ const MAX_TASK_SAMPLES = 20;
                                       
                                     
                                                         
+                                                                                           
+                                                        
                                        
                        
                          
                              
                                     
+                                                                                                
+                                                                                                          
+                                           
+                                                                                             
+                                               
+                                
                                                        
  
 
@@ -172,6 +200,10 @@ const MAX_TASK_SAMPLES = 20;
                        
                                                
                                       
+                                        
+                                                                                                 
+                                                                      
+                          
                                             
                                  
                                                                                      
@@ -361,6 +393,8 @@ export function createState()              {
     skippedLines: 0,
     accumState: new Map(),
     rddInfo: new Map(),
+    rddBlocks: new Map(),
+    rddBlockUpdates: 0,
     taskAccumStages: new Map(),
     pendingAdaptiveUpdates: new Map(),
     resolvedPlanExecutions: new Set(),
@@ -472,6 +506,7 @@ function snapshotEvidenceInputs(state             )                 {
 function appMessage(state             )                                                 {
   const evidenceInputs = snapshotEvidenceInputs(state);
   state.app .evidenceInputs = evidenceInputs;
+  state.app .rddBlockUpdates = state.rddBlockUpdates;
   return {
     type: 'app',
     data: { ...state.app , rddInfo: snapshotRddInfo(state.rddInfo) },
@@ -485,13 +520,30 @@ function taskRecordToSample(t            )                   {
   };
 }
 
+// One shared detail object per distinct failure, so a stage with thousands of failed attempts
+// holds each bounded stack excerpt once. Past the cap an attempt keeps only its reason tag.
+function internTaskFailure(stage             , endReason                                     )                           {
+  const detail = extractTaskFailureDetail(endReason);
+  if (!detail || !stage.failureDetails) return null;
+  const key = taskFailureKey(detail);
+  const known = stage.failureDetails.get(key);
+  if (known) return known;
+  if (stage.failureDetails.size >= MAX_FAILURE_DETAILS_PER_STAGE) return null;
+  stage.failureDetails.set(key, detail);
+  return detail;
+}
+
 export function accumulateTask(event                                    , state             )       {
   const stageId = event['Stage ID'];
   const stage = state.stages.get(stageId);
   if (!stage) return null;
   // Late TaskEnd for a stage whose StageCompleted already freed taskAttempts (finalizeStage): its
-  // stats are already baked into the finalized stage, don't re-add.
-  if (stage.taskAttempts === null) return null;
+  // stats are already baked into the finalized stage, don't re-add. The one exception is a losing
+  // speculative attempt, whose wasted time the finalized stage never saw.
+  if (stage.taskAttempts === null) {
+    accountLateSpeculativeLoser(event, stage);
+    return null;
+  }
 
   state.evidenceInputs.taskRecords++;
 
@@ -521,6 +573,7 @@ export function accumulateTask(event                                    , state
     launchTime: info['Launch Time'] ?? 0,
     finishTime: info['Finish Time'] ?? 0,
     reason: event['Task End Reason']?.['Reason'] ?? null,
+    failure: failed ? internTaskFailure(stage, event['Task End Reason']) : null,
     speculative: info['Speculative'] === true,
     host: info['Host'] ?? '',
     executorId: info['Executor ID'] ?? '',
@@ -546,6 +599,7 @@ export function accumulateTask(event                                    , state
 
   if (!existing) {
     stage.taskAttempts.set(key, record);
+    if (record.speculative && !record.failed) stage.speculativeWinners.add(key);
   } else if (existing.failed && !record.failed) {
     // A retry succeeded where the earlier attempt failed: the earlier attempt's time was wasted.
     // Spark marks only the speculative COPY's Speculative flag, never the original it raced, so
@@ -561,6 +615,7 @@ export function accumulateTask(event                                    , state
       }
     }
     stage.taskAttempts.set(key, record);
+    if (record.speculative) stage.speculativeWinners.add(key);
   } else {
     // Non-winning duplicate (both failed, or a race where a winner is
     // already recorded): its time is waste, its metrics are discarded.
@@ -579,6 +634,21 @@ export function accumulateTask(event                                    , state
   return null;
 }
 
+// Spark kills the losing copy of a speculative race only once the stage finishes ("Stage
+// cancelled: Stage finished"), so that loser's TaskEnd normally lands after StageCompleted. Count
+// its time as speculation waste, pairing it the same way accumulateTask does: the late attempt is
+// the speculative copy itself, or the original that a speculative winner beat. Every other stat
+// of a late attempt stays excluded, as the finalized stage already posted them.
+function accountLateSpeculativeLoser(event                                    , stage             )       {
+  const info = event['Task Info'];
+  if (info?.['Index'] == null) return;
+  const key = `${event['Stage Attempt ID'] ?? 0}:${info['Index']}`;
+  if (info['Speculative'] !== true && !stage.speculativeWinners.has(key)) return;
+  stage.speculationWasteMs += (info['Finish Time'] ?? 0) - (info['Launch Time'] ?? 0);
+  stage.speculationWastedAttempts++;
+  stage.lateSpeculationWaste = true;
+}
+
 export function resolvePlanTree(
   rootInfo               ,
   accumMap                     ,
@@ -802,11 +872,14 @@ export function submitStage(event                                           , st
     failureReasons: new Map(),
     stageFailureReason: null,
     taskAttempts: new Map(),
+    failureDetails: new Map(),
     retryTaskSamples: [],
     retryWasteMs: 0,
     wastedAttempts: 0,
     speculationWasteMs: 0,
     speculationWastedAttempts: 0,
+    speculativeWinners: new Set(),
+    lateSpeculationWaste: false,
     executorMetrics: new Map(),
   });
   mergeStageRddInfo(info, id, state);
@@ -826,11 +899,16 @@ export function mergeStageRddInfo(
     const prev = state.rddInfo.get(rddId);
     const stageIds = prev?.stageIds ?? new Set        ();
     stageIds.add(id);
+    // Block updates are the authoritative source once seen: never let a later RDD Info snapshot
+    // (0 on Spark 2.3+) replace them, nor the NONE level an unpersist() leaves on ancestor RDDs
+    // listed by later stages.
+    const fromBlocks = prev?.storageSource === 'blockUpdates';
+    const persisted = Boolean(sl['Use Disk'] || sl['Use Memory']);
     state.rddInfo.set(rddId, {
       id: rddId,
       name: rdd['Name'] ?? '',
       callsite: rdd['Callsite'] ?? '',
-      storageLevel: {
+      storageLevel: fromBlocks && !persisted ? prev.storageLevel : {
         useDisk: sl['Use Disk'] ?? false,
         useMemory: sl['Use Memory'] ?? false,
         deserialized: sl['Deserialized'] ?? false,
@@ -840,14 +918,88 @@ export function mergeStageRddInfo(
       // Merge forward, don't overwrite: an RDD cached for the first time in THIS stage legitimately
       // reports 0 (snapshot reflects BlockManager state at submission). Keep the last real value on
       // a resubmission instead of regressing to 0.
-      numCachedPartitions: rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
-      memorySize: rdd['Memory Size'] || prev?.memorySize || 0,
-      diskSize: rdd['Disk Size'] || prev?.diskSize || 0,
+      numCachedPartitions: fromBlocks ? prev.numCachedPartitions : rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
+      memorySize: fromBlocks ? prev.memorySize : rdd['Memory Size'] || prev?.memorySize || 0,
+      diskSize: fromBlocks ? prev.diskSize : rdd['Disk Size'] || prev?.diskSize || 0,
+      storageSource: prev?.storageSource ?? 'rddInfo',
       stageIds,
     });
   }
 }
 
+const RDD_BLOCK_ID = /^rdd_(\d+)_(\d+)$/;
+
+function dropRddBlock(rdd               , key        )       {
+  const prev = rdd.blocks.get(key);
+  if (!prev) return;
+  rdd.memorySize -= prev.memorySize;
+  rdd.diskSize -= prev.diskSize;
+  const replicas = (rdd.replicasByPartition.get(prev.partition) ?? 1) - 1;
+  if (replicas > 0) rdd.replicasByPartition.set(prev.partition, replicas);
+  else rdd.replicasByPartition.delete(prev.partition);
+  rdd.blocks.delete(key);
+}
+
+/**
+ * Folds one SparkListenerBlockUpdated into its RDD's live residency, then publishes the RDD's
+ * peak state to rddInfo: the most partitions resident at once, with the memory/disk bytes at the
+ * latest moment that peak held. A peak rather than the final state, because an unpersist() (or
+ * the app's own cleanup) removes every block before the log ends, and a final snapshot would
+ * read as "nothing was cached". Ties refresh, so a partition dropping from memory to disk after
+ * the peak still shows up in diskSize. Non-RDD blocks (broadcast, shuffle, task results) are ignored.
+ */
+export function recordBlockUpdate(event                                         , state             )       {
+  const info = event['Block Updated Info'];
+  const match = RDD_BLOCK_ID.exec(info['Block ID']);
+  if (!match) return null;
+  state.rddBlockUpdates++;
+  const rddId = Number(match[1]);
+  const partition = Number(match[2]);
+  const sl = info['Storage Level'] ?? {};
+  // Spark's StorageLevel.isValid: a removal or eviction reports level NONE.
+  const resident = Boolean(sl['Use Memory'] || sl['Use Disk']) && (sl['Replication'] ?? 1) > 0;
+
+  let rdd = state.rddBlocks.get(rddId);
+  if (!rdd) {
+    rdd = { blocks: new Map(), replicasByPartition: new Map(), memorySize: 0, diskSize: 0, peakCachedPartitions: 0 };
+    state.rddBlocks.set(rddId, rdd);
+  }
+  const key = `${partition}@${info['Block Manager ID']?.['Executor ID'] ?? ''}`;
+  dropRddBlock(rdd, key);
+  if (resident) {
+    // Sizes count only where the level says the block lives, as Spark's AppStatusListener does: a
+    // drop from memory to disk reports Use Memory false but still carries the dropped bytes as
+    // Memory Size (BlockManager reports max(memSize, droppedMemorySize)).
+    const block = {
+      partition,
+      memorySize: sl['Use Memory'] ? info['Memory Size'] ?? 0 : 0,
+      diskSize: sl['Use Disk'] ? info['Disk Size'] ?? 0 : 0,
+    };
+    rdd.blocks.set(key, block);
+    rdd.memorySize += block.memorySize;
+    rdd.diskSize += block.diskSize;
+    rdd.replicasByPartition.set(partition, (rdd.replicasByPartition.get(partition) ?? 0) + 1);
+  }
+
+  const record = state.rddInfo.get(rddId) ?? {
+    // A block reported before any stage listed its RDD: name and partition count arrive with
+    // the next StageSubmitted (mergeStageRddInfo keeps the block-derived sizes).
+    id: rddId, name: '', callsite: '',
+    storageLevel: { useDisk: Boolean(sl['Use Disk']), useMemory: Boolean(sl['Use Memory']), deserialized: false, replication: sl['Replication'] ?? 1 },
+    numPartitions: 0, numCachedPartitions: 0, memorySize: 0, diskSize: 0,
+    storageSource: 'blockUpdates'         , stageIds: new Set        (),
+  };
+  record.storageSource = 'blockUpdates';
+  if (rdd.replicasByPartition.size >= rdd.peakCachedPartitions) {
+    rdd.peakCachedPartitions = rdd.replicasByPartition.size;
+    record.numCachedPartitions = rdd.peakCachedPartitions;
+    record.memorySize = rdd.memorySize;
+    record.diskSize = rdd.diskSize;
+  }
+  state.rddInfo.set(rddId, record);
+  return null;
+}
+
 // Spark logs these AFTER SparkListenerStageCompleted, so finalizeStage has already posted the
 // stage with an empty executorMetrics Map. Keep accumulating worker-side, then re-post all maps
 // once via `stageExecutorMetrics` just before `done` so main-thread stages are patched before analyze().
@@ -961,6 +1113,14 @@ export function removeExecutor(event
     reason: event['Removed Reason'] ?? '',
   };
   state.executors.removed.push(ev);
+  // Blocks lost with their executor get no BlockUpdated; drop them as Spark's AppStatusListener
+  // does, so a partition re-cached elsewhere isn't counted twice.
+  const suffix = `@${ev.executorId}`;
+  for (const rdd of state.rddBlocks.values()) {
+    for (const key of rdd.blocks.keys()) {
+      if (key.endsWith(suffix)) dropRddBlock(rdd, key);
+    }
+  }
   return { type: 'executor', data: ev };
 }
 
@@ -1037,6 +1197,9 @@ export function processEvent(event            , state             )          {
     case 'SparkListenerExecutorRemoved':
       return removeExecutor(event, state);
 
+    case 'SparkListenerBlockUpdated':
+      return recordBlockUpdate(event, state);
+
     default:
       return assertNever(event);
   }
@@ -1194,7 +1357,14 @@ export function dispatchLine(
   parseAndDispatch(line, state, emit);
 }
 
+// With logBlockUpdates on, most BlockUpdated lines are broadcast/shuffle blocks recordBlockUpdate
+// ignores. Spark writes "Event" first and "Block ID" as a plain string, so those lines can be
+// dropped on a substring test before paying for JSON.parse.
+const BLOCK_UPDATED_PREFIX = '{"Event":"SparkListenerBlockUpdated",';
+const RDD_BLOCK_ID_FRAGMENT = '"Block ID":"rdd_';
+
 function parseAndDispatch(line        , state             , emit                        )       {
+  if (line.startsWith(BLOCK_UPDATED_PREFIX) && !line.includes(RDD_BLOCK_ID_FRAGMENT)) return;
   let parsed         ;
   try {
     parsed = parseTaskEnd(line) ?? JSON.parse(stripPlanDescription(line));
@@ -1244,11 +1414,26 @@ export function collectStageExecutorMetrics(state             )
   return out;
 }
 
+// Speculation totals of every stage a late TaskEnd added waste to (accountLateSpeculativeLoser),
+// re-posted once before `done`: the stage message posted at completion carried the earlier totals.
+export function collectLateSpeculationWaste(
+  state             ,
+)                                                                                 {
+  const out = new Map                                                                           ();
+  for (const [id, stage] of state.stages) {
+    if (stage.lateSpeculationWaste) {
+      out.set(id, { speculationWasteMs: stage.speculationWasteMs, speculationWastedAttempts: stage.speculationWastedAttempts });
+    }
+  }
+  return out;
+}
+
 export function emitParseCompletion(state             , emit                        , linesProcessed        )       {
   // Executions that never ended keep their latest AQE update, as they did before it was deferred.
   for (const executionId of [...state.pendingAdaptiveUpdates.keys()]) flushAdaptiveUpdate(executionId, state, emit);
   emit({ type: 'progress', pct: 1, linesProcessed });
   emit({ type: 'runAggregates', data: computeRunAggregates(state.taskStore) });
+  emit({ type: 'stageSpeculationWaste', data: collectLateSpeculationWaste(state) });
   emit({ type: 'stageExecutorMetrics', data: collectStageExecutorMetrics(state) });
   emit(appMessage(state));
   emit({ type: 'done', skippedLines: state.skippedLines });

package/vendor-core/event-schemas.js

@@ -209,6 +209,26 @@ export const StageSubmittedEventSchema = z.object({
   }),
 });
 
+// recordBlockUpdate. Written only with spark.eventLog.logBlockUpdates.enabled=true; one event per
+// block status change reported to the driver's BlockManagerMaster (a removal or eviction carries
+// storage level NONE with zero sizes). Only 'rdd_<rddId>_<partition>' blocks are read.
+export const BlockUpdatedEventSchema = z.object({
+  Event: z.literal('SparkListenerBlockUpdated'),
+  'Block Updated Info': z.object({
+    'Block Manager ID': z.object({
+      'Executor ID': z.string().optional(),
+    }).optional(),
+    'Block ID': z.string(),
+    'Storage Level': z.object({
+      'Use Disk': z.boolean().optional(),
+      'Use Memory': z.boolean().optional(),
+      Replication: z.number().optional(),
+    }).optional(),
+    'Memory Size': z.number().optional(),
+    'Disk Size': z.number().optional(),
+  }),
+});
+
 // inline StageCompleted case.
 export const StageCompletedEventSchema = z.object({
   Event: z.literal('SparkListenerStageCompleted'),
@@ -238,6 +258,14 @@ export const TaskEndEventSchema = z.object({
   'Stage Attempt ID': z.number().optional(),
   'Task End Reason': z.object({
     Reason: z.string().optional(),
+    // The error behind a failed attempt, read by extractTaskFailureDetail (task-failure.ts), which
+    // keeps only string values: unknown here so an unexpected shape drops the detail, never the task.
+    'Class Name': z.unknown().optional(),
+    Description: z.unknown().optional(),
+    'Full Stack Trace': z.unknown().optional(),
+    'Loss Reason': z.unknown().optional(),
+    Message: z.unknown().optional(),
+    'Kill Reason': z.unknown().optional(),
   }).optional(),
   'Task Info': z.object({
     'Launch Time': z.number().optional(),
@@ -337,5 +365,6 @@ export const SparkEventSchema = z.discriminatedUnion('Event', [
   DriverAccumUpdatesEventSchema,
   ExecutorAddedEventSchema,
   ExecutorRemovedEventSchema,
+  BlockUpdatedEventSchema,
 ]);