sparkforensics-server 0.4.0 → 0.5.1

package/vendor-core/event-handlers.js

@@ -18,12 +18,14 @@ import {
                   
                      
 } from './event-schemas.js';
+import { DELTA_COMMANDS } from './delta-commands.js';
 import { assertNever } from './assert-never.js';
 import { finalizeStage } from './stage-quantiles.js';
 import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
 import { computeRunAggregates } from './run-aggregates.js';
+import { parseSparkMemoryMB } from './spark-memory.js';
              
-                                                                                        
+                                                                                                            
                  
 
 // Internal parser-state shapes: the real runtime objects the handlers build and mutate, not the
@@ -129,6 +131,8 @@ const MAX_TASK_SAMPLES = 20;
                           
                      
                       
+                                                                                                
+                               
  
 
                        
@@ -149,12 +153,29 @@ const MAX_TASK_SAMPLES = 20;
                           
                      
                       
+                                                                      
+                               
                                 
                       
                                   
                            
                                       
                                     
+                                                                                                        
+                         
+                                                                                                  
+                                                                                       
+                        
+                              
+                                                                                       
+                                             
+                                                                                       
+                                                                                             
+                                                                                              
+                                                     
+                                             
+                                                              
+                                  
                                                         
                                                                                            
                                                         
@@ -178,6 +199,9 @@ const MAX_TASK_SAMPLES = 20;
                     
                          
                      
+                           
+                                                                                  
+                            
                                                                                          
                                       
                                                                                                   
@@ -198,6 +222,9 @@ const MAX_TASK_SAMPLES = 20;
                          
                                                                               
                        
+                                                                                               
+                                                                                  
+                                   
                                                
                                       
                                         
@@ -256,6 +283,10 @@ export function buildChunkDecoder() {
   // line again after every slice scanned about 10 GB on a 100 MB line.
   let pendingIsSqlEvent = false;
   let keyTail = '';
+  // Where the pending line's plan-description value starts, once its key is found but too little
+  // of the value has arrived to tell whether it is a small Delta command description (see
+  // descriptionDisposition); -1 otherwise.
+  let undecidedValueStart = -1;
   let skippingPlanDescription = false;
   // Length of the backslash run the skipped bytes ended with, which escapes a quote at the start
   // of the next chunk when odd.
@@ -289,11 +320,24 @@ export function buildChunkDecoder() {
       keyTail = '';
       appended = pending; // nothing of it was searched while it was shorter than the prefix
     }
-    const keyAt = findPlanKey(appended);
-    if (keyAt === -1) return; // the key may still arrive in a later chunk
+    let valueStart = undecidedValueStart;
+    if (valueStart === -1) {
+      const keyAt = findPlanKey(appended);
+      if (keyAt === -1) return; // the key may still arrive in a later chunk
+      valueStart = keyAt + PLAN_DESCRIPTION_KEY.length;
+    }
+    if (closingQuoteIndex(pending, valueStart) !== -1) { // complete: stripPlanDescription handles it
+      undecidedValueStart = -1;
+      pendingSettled = true;
+      return;
+    }
+    // A small kept description is not skipped: the line completes in a later chunk and
+    // stripPlanDescription keeps its arguments line.
+    const disposition = descriptionDisposition(pending, valueStart);
+    if (disposition === 'wait') { undecidedValueStart = valueStart; return; }
+    undecidedValueStart = -1;
     pendingSettled = true;
-    const valueStart = keyAt + PLAN_DESCRIPTION_KEY.length;
-    if (closingQuoteIndex(pending, valueStart) !== -1) return; // complete: stripPlanDescription empties it
+    if (disposition === 'keep') return;
     // Only a chunk whose last byte is a backslash carries a run over; any other last byte (such
     // as part of a split multibyte char) ends it.
     let run = 0;
@@ -355,6 +399,7 @@ export function buildChunkDecoder() {
       pending = pendingHead = appended = start < text.length ? text.substring(start) : '';
       pendingSettled = false;
       pendingIsSqlEvent = false;
+      undecidedValueStart = -1;
     }
     if (!pendingSettled && pending !== '') settlePending(buffer[buffer.length - 1], appended);
   }
@@ -391,6 +436,7 @@ export function createState()              {
     jobs: new Map(),
     executors: { added: [], removed: [] },
     skippedLines: 0,
+    unreadableSqlStarts: new Set(),
     accumState: new Map(),
     rddInfo: new Map(),
     rddBlocks: new Map(),
@@ -432,23 +478,7 @@ export function normalizeSparkProperties(
   return map;
 }
 
-// Parse a Spark memory-size string to MiB. Spark's JVM-memory configs use bytesConf(ByteUnit.MiB),
-// so a bare number means MiB. A k/m/g/t suffix sets the unit (trailing "b" redundant); a lone "b"
-// ("10b") means bytes.
-export function parseSparkMemoryMB(value         )                {
-  if (value == null) return null;
-  const m = String(value).trim().toLowerCase().match(/^([\d.]+)\s*([kmgt]?)(b?)$/);
-  if (!m) return null;
-  const n = parseFloat(m[1]);
-  if (!Number.isFinite(n)) return null;
-  switch (m[2]) {
-    case 'k': return Math.round(n / 1024);
-    case 'g': return Math.round(n * 1024);
-    case 't': return Math.round(n * 1024 * 1024);
-    case 'm': return Math.round(n);
-    default: return m[3] === 'b' ? Math.round(n / (1024 * 1024)) : Math.round(n);
-  }
-}
+export { parseSparkMemoryMB };
 
 // Derive an allocated-resource summary from the Spark config map. Absent keys degrade to null,
 // not guessed defaults.
@@ -533,29 +563,11 @@ function internTaskFailure(stage             , endReason
   return detail;
 }
 
-export function accumulateTask(event                                    , state             )       {
-  const stageId = event['Stage ID'];
-  const stage = state.stages.get(stageId);
-  if (!stage) return null;
-  // Late TaskEnd for a stage whose StageCompleted already freed taskAttempts (finalizeStage): its
-  // stats are already baked into the finalized stage, don't re-add. The one exception is a losing
-  // speculative attempt, whose wasted time the finalized stage never saw.
-  if (stage.taskAttempts === null) {
-    accountLateSpeculativeLoser(event, stage);
-    return null;
-  }
-
-  state.evidenceInputs.taskRecords++;
-
-  const accumulables = event['Task Info']?.Accumulables ?? [];
-  for (const acc of accumulables) {
-    if (!state.taskAccumStages.has(acc.ID)) state.taskAccumStages.set(acc.ID, new Set());
-    state.taskAccumStages.get(acc.ID) .add(stageId);
-  }
+// 'Task Info' and its Failed/Killed/Speculative fields are optional in the schema; Partial<>
+// lets the {} fallback type-check while reads below default via ??/||.
+                                                                                         
 
-  // 'Task Info' and its Failed/Killed/Speculative fields are optional in the schema; Partial<>
-  // lets the {} fallback type-check while reads below default via ??/||.
-                                                                                           
+function taskRecordOf(event                                    , failure                          )             {
   const info              = event['Task Info'] ?? {};
   const m = event['Task Metrics'] ?? {};
   const sr = m['Shuffle Read Metrics'] ?? {};
@@ -566,14 +578,14 @@ export function accumulateTask(event                                    , state
   const duration = (info['Finish Time'] ?? 0) - (info['Launch Time'] ?? 0);
   const failed = !!(info['Failed'] || info['Killed']);
 
-  const record             = {
+  return {
     duration, failed,
     taskId: info['Task ID'] ?? null,
     attemptNumber: info['Attempt Number'] ?? 0,
     launchTime: info['Launch Time'] ?? 0,
     finishTime: info['Finish Time'] ?? 0,
     reason: event['Task End Reason']?.['Reason'] ?? null,
-    failure: failed ? internTaskFailure(stage, event['Task End Reason']) : null,
+    failure,
     speculative: info['Speculative'] === true,
     host: info['Host'] ?? '',
     executorId: info['Executor ID'] ?? '',
@@ -589,7 +601,37 @@ export function accumulateTask(event                                    , state
     executorCpuTime: m['Executor CPU Time'] ?? 0,
     inputBytes: inp['Bytes Read'] ?? 0,
     outputBytes: out['Bytes Written'] ?? 0,
+    outputRecords: out['Records Written'] ?? null,
   };
+}
+
+export function accumulateTask(event                                    , state             )       {
+  const stageId = event['Stage ID'];
+  const stage = state.stages.get(stageId);
+  if (!stage) return null;
+  // Late TaskEnd for a stage whose StageCompleted already freed taskAttempts (finalizeStage): the
+  // finalized stage's figures stay as posted. A losing speculative attempt adds the wasted time the
+  // finalized stage never saw; any other task of a failed or earlier attempt is work only the
+  // metrics block reads, from lateAttemptWork.
+  if (stage.taskAttempts === null) {
+    const earlierAttempt = (event['Stage Attempt ID'] ?? 0) !== stage.stageAttemptId;
+    if (!accountLateSpeculativeLoser(event, stage) && (stage.stageFailureReason != null || earlierAttempt)) {
+      stage.lateAttemptWork = mergeAttemptTotals(taskAttemptTotals(taskRecordOf(event, null)), stage.lateAttemptWork);
+    }
+    return null;
+  }
+
+  state.evidenceInputs.taskRecords++;
+
+  const accumulables = event['Task Info']?.Accumulables ?? [];
+  for (const acc of accumulables) {
+    if (!state.taskAccumStages.has(acc.ID)) state.taskAccumStages.set(acc.ID, new Set());
+    state.taskAccumStages.get(acc.ID) .add(stageId);
+  }
+
+  const info              = event['Task Info'] ?? {};
+  const failed = !!(info['Failed'] || info['Killed']);
+  const record = taskRecordOf(event, failed ? internTaskFailure(stage, event['Task End Reason']) : null);
 
   // Dedupe only when Index is present (always true for real logs). Without it every event is a
   // distinct task, preserving behavior for fixtures that omit Index.
@@ -614,9 +656,11 @@ export function accumulateTask(event                                    , state
         stage.retryTaskSamples.push(taskRecordToSample(existing));
       }
     }
+    stage.lateAttemptWork = mergeAttemptTotals(discardedAttemptTotals(existing), stage.lateAttemptWork);
     stage.taskAttempts.set(key, record);
     if (record.speculative) stage.speculativeWinners.add(key);
   } else {
+    stage.lateAttemptWork = mergeAttemptTotals(discardedAttemptTotals(record), stage.lateAttemptWork);
     // Non-winning duplicate (both failed, or a race where a winner is
     // already recorded): its time is waste, its metrics are discarded.
     if (existing.speculative || record.speculative) {
@@ -639,14 +683,16 @@ export function accumulateTask(event                                    , state
 // its time as speculation waste, pairing it the same way accumulateTask does: the late attempt is
 // the speculative copy itself, or the original that a speculative winner beat. Every other stat
 // of a late attempt stays excluded, as the finalized stage already posted them.
-function accountLateSpeculativeLoser(event                                    , stage             )       {
+function accountLateSpeculativeLoser(event                                    , stage             )          {
   const info = event['Task Info'];
-  if (info?.['Index'] == null) return;
+  if (info?.['Index'] == null) return false;
   const key = `${event['Stage Attempt ID'] ?? 0}:${info['Index']}`;
-  if (info['Speculative'] !== true && !stage.speculativeWinners.has(key)) return;
+  if (info['Speculative'] !== true && !stage.speculativeWinners.has(key)) return false;
   stage.speculationWasteMs += (info['Finish Time'] ?? 0) - (info['Launch Time'] ?? 0);
   stage.speculationWastedAttempts++;
   stage.lateSpeculationWaste = true;
+  stage.lateAttemptWork = mergeAttemptTotals(discardedAttemptTotals(taskRecordOf(event, null)), stage.lateAttemptWork);
+  return true;
 }
 
 export function resolvePlanTree(
@@ -853,10 +899,63 @@ export function endJob(event                                   , state
   return { type: 'job', data: { ...job } };
 }
 
+const SUMMED_ATTEMPT_FIELDS = [
+  'taskCount', 'failedTasks', 'wastedAttempts', 'executorRunTime', 'executorCpuTime', 'jvmGCTime',
+  'memoryBytesSpilled', 'diskBytesSpilled', 'shuffleReadBytes', 'shuffleWriteBytes', 'inputBytes', 'outputBytes',
+]         ;
+
+const addNullable = (a               , b               )                => (a == null ? b : b == null ? a : a + b);
+
+function mergeAttemptTotals(a                    , b                           )                     {
+  if (b == null) return a;
+  const totals = {
+    outputRecords: addNullable(a.outputRecords, b.outputRecords),
+    peakExecutionMemoryMax: Math.max(a.peakExecutionMemoryMax, b.peakExecutionMemoryMax),
+    durationMs: addNullable(a.durationMs, b.durationMs),
+  }                      ;
+  for (const field of SUMMED_ATTEMPT_FIELDS) totals[field] = a[field] + b[field];
+  return totals;
+}
+
+// One task's work; a late task adds no stage duration.
+function taskAttemptTotals(t            )                     {
+  return {
+    taskCount: 1, failedTasks: t.failed ? 1 : 0, wastedAttempts: 0,
+    executorRunTime: t.executorRunTime, executorCpuTime: t.executorCpuTime, jvmGCTime: t.gcTime,
+    memoryBytesSpilled: t.memSpilled, diskBytesSpilled: t.diskSpilled,
+    shuffleReadBytes: t.shuffleRead, shuffleWriteBytes: t.shuffleWrite,
+    inputBytes: t.inputBytes, outputBytes: t.outputBytes, outputRecords: t.outputRecords,
+    peakExecutionMemoryMax: t.peakExecMem, durationMs: null,
+  };
+}
+
+// A task attempt the stage's figures drop because another attempt of the same task won (a retry
+// after a failure, a speculative twin): its CPU, run time and I/O were still spent, so the
+// metrics block counts them, but the task itself is already counted once.
+function discardedAttemptTotals(t            )                     {
+  return { ...taskAttemptTotals(t), taskCount: 0, failedTasks: 0 };
+}
+
+// The replaced record's finalized attempt added to the attempts it had already folded. An attempt resubmitted before its StageCompleted was never finalized, so its
+// tasks are not counted.
+function foldEarlierAttempts(replaced                         )                            {
+  if (!replaced) return null;
+  if (replaced.taskAttempts !== null) return replaced.earlierAttempts;
+  const attempt                     = {
+    ...Object.fromEntries(SUMMED_ATTEMPT_FIELDS.map((field) => [field, replaced[field]]))                                                                  ,
+    outputRecords: replaced.outputRecords,
+    peakExecutionMemoryMax: replaced.peakExecutionMemoryMax ?? 0,
+    durationMs: replaced.submittedAt > 0 && replaced.completedAt >= replaced.submittedAt
+      ? replaced.completedAt - replaced.submittedAt : null,
+  };
+  return mergeAttemptTotals(attempt, replaced.earlierAttempts);
+}
+
 export function submitStage(event                                           , state             )       {
   state.evidenceInputs.stageSubmissions++;
   const info = event['Stage Info'];
   const id = info['Stage ID'];
+  const replaced = state.stages.get(id);
   state.stages.set(id, {
     id, name: info['Stage Name'] ?? '', details: info['Details'] ?? '',
     submittedAt: info['Submission Time'] ?? 0, completedAt: 0,
@@ -864,13 +963,18 @@ export function submitStage(event                                           , st
     shuffleReadBytes: 0, shuffleWriteBytes: 0, fetchWaitTime: 0,
     memoryBytesSpilled: 0, diskBytesSpilled: 0,
     jvmGCTime: 0, executorRunTime: 0, executorCpuTime: 0,
-    inputBytes: 0, outputBytes: 0,
+    inputBytes: 0, outputBytes: 0, outputRecords: null,
     sqlExecutionId: state.stageToSqlExec.get(id) ?? null,
     parentIds: info['Parent IDs'] ?? [],
     hostStats: new Map(),
     speculativeTasks: 0,
     failureReasons: new Map(),
     stageFailureReason: null,
+    stageAttemptId: info['Stage Attempt ID'] ?? 0,
+    stageAttempts: (replaced?.stageAttempts ?? 0) + 1,
+    failedStageAttempts: replaced?.failedStageAttempts ?? 0,
+    earlierAttempts: foldEarlierAttempts(replaced),
+    lateAttemptWork: replaced?.lateAttemptWork ?? null,
     taskAttempts: new Map(),
     failureDetails: new Map(),
     retryTaskSamples: [],
@@ -1027,6 +1131,10 @@ export function startSqlExecution(event
     sparkPlanInfo,
     hadAdaptiveUpdate: false,
   };
+  if (event.rootExecutionId !== undefined) exec.rootExecutionId = event.rootExecutionId;
+  if (event.physicalPlanDescription && KEPT_ARGUMENTS.test(event.physicalPlanDescription)) {
+    exec.commandArguments = event.physicalPlanDescription;
+  }
   state.sqlExecutions.set(exec.id, exec);
   // A restarted execution carries a new plan: its next end must resolve it again.
   state.resolvedPlanExecutions.delete(exec.id);
@@ -1164,6 +1272,8 @@ export function processEvent(event            , state             )          {
       // every stage-duration figure becomes the epoch timestamp itself (a "47-year" stage).
       if (!stage.submittedAt && info['Submission Time'] != null) stage.submittedAt = info['Submission Time'];
       stage.stageFailureReason = info['Failure Reason'] ?? null;
+      // A duplicate StageCompleted of an already-finalized attempt is not another failed attempt.
+      if (stage.stageFailureReason != null && stage.taskAttempts !== null) stage.failedStageAttempts++;
       // finalizeStage keeps its `stage` parameter typed as a loose Record (see that module); bridge
       // StageRecord's more precise shape across that boundary with an explicit cast.
       return finalizeStage(
@@ -1211,13 +1321,32 @@ const KNOWN_EVENT_TYPES                      = new Set(
 );
 
 // SQLExecutionStart and SQLAdaptiveExecutionUpdate carry `physicalPlanDescription`, Spark's text
-// rendering of the plan. Nothing reads it (the plan tree comes from sparkPlanInfo), yet on a real
-// 3.5 GB log it was 72% of the AQE-update bytes, which were themselves 73% of the log. Cutting its
+// rendering of the plan. The plan tree comes from sparkPlanInfo, yet on a real 3.5 GB log the
+// description was 72% of the AQE-update bytes, which were themselves 73% of the log. Cutting its
 // string value out before JSON.parse halves the parse cost of those lines.
+//
+// One exception: a Delta write command's node prints only its bare name in sparkPlanInfo, and its
+// target table appears only on its `Arguments:` line of the description. When the description is
+// that of one of KEPT_ARGUMENTS_COMMANDS (checked on its first bytes, so no other line pays for it),
+// the value is cut down to that one line, never kept whole. A value that spans decoder chunks is
+// still dropped unread (buildChunkDecoder), so the write target then falls back to its child paths.
 const SQL_UI_EVENT_PREFIX = '{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQL';
 const PLAN_DESCRIPTION_KEY = '"physicalPlanDescription":"';
 
-const QUOTE = 0x22, BACKSLASH = 0x5c, NEWLINE = 0x0a;
+const QUOTE = 0x22, BACKSLASH = 0x5c, NEWLINE = 0x0a, LOWER_N = 0x6e;
+
+// Delta commands whose root node carries no target in sparkPlanInfo. The description of a
+// command execution starts `== Physical Plan ==\nExecute <Command> (1)` (JSON-escaped newline).
+const KEPT_ARGUMENTS_COMMANDS                      = new Set(DELTA_COMMANDS);
+// Those whose description is just the command (no child plan printed under it), so it is small
+// enough to decode whole when it spans decoder chunks. SaveIntoDataSourceCommand prints the whole
+// query under it, which can be as large as any other plan description.
+const SMALL_DESCRIPTION_COMMANDS                      = new Set(
+  [...KEPT_ARGUMENTS_COMMANDS].filter((command) => command !== 'SaveIntoDataSourceCommand'),
+);
+const PLAN_HEADER = '== Physical Plan ==\\nExecute ';
+// The kept form once JSON-parsed: `<Command>\nArguments: <line>`.
+const KEPT_ARGUMENTS = /^[A-Za-z]+\nArguments: /;
 
 // Index of the closing quote of the JSON string whose content starts at `valueStart`: the first
 // quote preceded by an even number of backslashes. -1 when the string is unterminated.
@@ -1242,7 +1371,48 @@ function closingQuoteAt(buf            , limit        , carried        )
   return -1;
 }
 
-// Returns `line` with the physicalPlanDescription string value emptied, or `line` unchanged when
+// The command named by a description that starts at text[valueStart] with PLAN_HEADER, when text
+// holds the whole name; null otherwise.
+function describedCommand(text        , valueStart        )                {
+  if (!text.startsWith(PLAN_HEADER, valueStart)) return null;
+  const nameStart = valueStart + PLAN_HEADER.length;
+  let nameEnd = nameStart;
+  while (nameEnd < text.length && /[A-Za-z]/.test(text[nameEnd])) nameEnd++;
+  return nameEnd < text.length ? text.slice(nameStart, nameEnd) : null;
+}
+
+// What to do with an open description value whose first bytes are text[valueStart...]: 'keep' it
+// (a small Delta command description), 'skip' it, or 'wait' when too little has arrived to tell.
+function descriptionDisposition(text        , valueStart        )                           {
+  const seen = text.slice(valueStart, valueStart + PLAN_HEADER.length);
+  if (!PLAN_HEADER.startsWith(seen)) return 'skip';
+  if (seen.length < PLAN_HEADER.length) return 'wait';
+  const command = describedCommand(text, valueStart);
+  if (command === null) return 'wait'; // the command name is still arriving
+  return SMALL_DESCRIPTION_COMMANDS.has(command) ? 'keep' : 'skip';
+}
+
+// The `<Command>\nArguments: <line>` part (still JSON-escaped) of the root command of a plan
+// description whose value spans line[valueStart, quote), or null when it is not one of
+// KEPT_ARGUMENTS_COMMANDS. SaveIntoDataSourceCommand is kept only for the Delta source: its other
+// providers (JDBC) print option maps that this tool has no use for.
+function keptCommandArguments(line        , valueStart        , quote        )                {
+  const command = describedCommand(line, valueStart);
+  if (command === null || !KEPT_ARGUMENTS_COMMANDS.has(command)) return null;
+  const key = `${command}\\nArguments: `;
+  const at = line.indexOf(key, valueStart + PLAN_HEADER.length + command.length);
+  if (at === -1 || at >= quote) return null;
+  // The line ends at the first escaped newline: a backslash starts a two-char escape.
+  let end = at + key.length;
+  while (end < quote && !(line.charCodeAt(end) === BACKSLASH && line.charCodeAt(end + 1) === LOWER_N)) {
+    end += line.charCodeAt(end) === BACKSLASH ? 2 : 1;
+  }
+  const kept = line.slice(at, Math.min(end, quote));
+  return command === 'SaveIntoDataSourceCommand' && !kept.includes('DeltaDataSource') ? null : kept;
+}
+
+// Returns `line` with the physicalPlanDescription string value emptied (or cut to the kept
+// arguments line, see keptCommandArguments), or `line` unchanged when
 // the key isn't found in Spark's compact form. The key pattern can't match inside another JSON
 // string: there its quotes would be backslash-escaped. buildChunkDecoder already empties a value
 // that spans chunks, so here the value is empty or within one chunk.
@@ -1255,7 +1425,7 @@ export function stripPlanDescription(line        )         {
   const quote = closingQuoteIndex(line, valueStart);
   // Unterminated string (a truncated line): leave it for JSON.parse to reject.
   if (quote === -1) return line;
-  return line.slice(0, valueStart) + line.slice(quote);
+  return line.slice(0, valueStart) + (keptCommandArguments(line, valueStart, quote) ?? '') + line.slice(quote);
 }
 
 const TASK_END_PREFIX = '{"Event":"SparkListenerTaskEnd",';
@@ -1363,6 +1533,18 @@ export function dispatchLine(
 const BLOCK_UPDATED_PREFIX = '{"Event":"SparkListenerBlockUpdated",';
 const RDD_BLOCK_ID_FRAGMENT = '"Block ID":"rdd_';
 
+const SQL_START_EVENT = 'org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart';
+const SQL_START_ID = /"executionId":(\d+)/;
+
+// Records the execution id of a skipped SQL start line, when the line's head still names it: a
+// line cut off mid-plan fails JSON.parse but keeps its leading fields.
+function noteUnreadableSqlStart(line        , state             )       {
+  const head = line.slice(0, 300);
+  if (!head.includes(`${SQL_START_EVENT}"`)) return;
+  const id = SQL_START_ID.exec(head);
+  if (id) state.unreadableSqlStarts.add(Number(id[1]));
+}
+
 function parseAndDispatch(line        , state             , emit                        )       {
   if (line.startsWith(BLOCK_UPDATED_PREFIX) && !line.includes(RDD_BLOCK_ID_FRAGMENT)) return;
   let parsed         ;
@@ -1370,6 +1552,7 @@ function parseAndDispatch(line        , state             , emit
     parsed = parseTaskEnd(line) ?? JSON.parse(stripPlanDescription(line));
   } catch {
     state.skippedLines++;
+    noteUnreadableSqlStart(line, state);
     return;
   }
   // A real Spark event log carries many event types this tool never modeled (BlockManagerAdded,
@@ -1382,6 +1565,7 @@ function parseAndDispatch(line        , state             , emit
   const result = SparkEventSchema.safeParse(parsed);
   if (!result.success) {
     state.skippedLines++;
+    if (eventType === SQL_START_EVENT) noteUnreadableSqlStart(line, state);
     return;
   }
   if (result.data.Event === 'org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd') {
@@ -1428,14 +1612,29 @@ export function collectLateSpeculationWaste(
   return out;
 }
 
+// Late work of every stage a failed attempt's late TaskEnd added to, re-posted once before `done`:
+// the stage message posted at completion predates it.
+export function collectLateAttemptWork(state             )                                  {
+  const out = new Map                            ();
+  for (const [id, stage] of state.stages) {
+    if (stage.lateAttemptWork != null) out.set(id, stage.lateAttemptWork);
+  }
+  return out;
+}
+
 export function emitParseCompletion(state             , emit                        , linesProcessed        )       {
   // Executions that never ended keep their latest AQE update, as they did before it was deferred.
   for (const executionId of [...state.pendingAdaptiveUpdates.keys()]) flushAdaptiveUpdate(executionId, state, emit);
   emit({ type: 'progress', pct: 1, linesProcessed });
+  emit({ type: 'stageLateAttemptWork', data: collectLateAttemptWork(state) });
   emit({ type: 'runAggregates', data: computeRunAggregates(state.taskStore) });
   emit({ type: 'stageSpeculationWaste', data: collectLateSpeculationWaste(state) });
   emit({ type: 'stageExecutorMetrics', data: collectStageExecutorMetrics(state) });
   emit(appMessage(state));
-  emit({ type: 'done', skippedLines: state.skippedLines });
+  emit({
+    type: 'done',
+    skippedLines: state.skippedLines,
+    ...(state.unreadableSqlStarts.size > 0 ? { unreadableSqlExecutions: [...state.unreadableSqlStarts].sort((a, b) => a - b) } : {}),
+  });
   state.accumState.clear();
 }

package/vendor-core/event-schemas.js

@@ -201,6 +201,7 @@ export const StageSubmittedEventSchema = z.object({
   Event: z.literal('SparkListenerStageSubmitted'),
   'Stage Info': z.object({
     'Stage ID': z.number(),
+    'Stage Attempt ID': z.number().optional(),
     'Stage Name': z.string().optional(),
     Details: z.string().optional(),
     'Submission Time': z.number().optional(),
@@ -311,6 +312,7 @@ export const TaskEndEventSchema = z.object({
     }).optional(),
     'Output Metrics': z.object({
       'Bytes Written': z.number().optional(),
+      'Records Written': z.number().optional(),
     }).optional(),
   }).optional(),
 });
@@ -319,13 +321,15 @@ export const TaskEndEventSchema = z.object({
 export const SqlExecutionStartEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart'),
   executionId: z.number(),
+  rootExecutionId: z.number().optional(),
   description: z.string().optional(),
+  physicalPlanDescription: z.string().optional(),
   time: z.number(),
   sparkPlanInfo: SparkPlanInfoFieldSchema,
 });
 
 // applyAdaptiveExecutionUpdate: AQE re-plans mid-query and re-emits sparkPlanInfo for the same
-// executionId; last write wins. Neither SQL schema declares physicalPlanDescription: nothing reads
+// executionId; last write wins. This schema does not declare physicalPlanDescription: nothing reads
 // it, so zod strips it (dispatchLine already empties it before JSON.parse, see stripPlanDescription).
 export const SqlAdaptiveExecutionUpdateEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate'),