sparkforensics-cli 0.1.0

package/vendor-core/event-handlers.js

@@ -0,0 +1,906 @@
+import { z } from 'zod';
+import {
+  SparkEventSchema,
+  ApplicationStartEventSchema,
+  EnvironmentUpdateEventSchema,
+  JobStartEventSchema,
+  JobEndEventSchema,
+  StageSubmittedEventSchema,
+  StageExecutorMetricsEventSchema,
+  TaskEndEventSchema,
+  SqlExecutionStartEventSchema,
+  SqlAdaptiveExecutionUpdateEventSchema,
+  SqlExecutionEndEventSchema,
+  DriverAccumUpdatesEventSchema,
+  ExecutorAddedEventSchema,
+  ExecutorRemovedEventSchema,
+                  
+                     
+} from './event-schemas.js';
+import { assertNever } from './assert-never.js';
+import { finalizeStage } from './stage-quantiles.js';
+import { computeRunAggregates } from './run-aggregates.js';
+             
+                                                                                        
+                 
+
+// ---------------------------------------------------------------------------
+// Internal parser-state shapes. These describe the real runtime objects the
+// handlers below build and mutate; they are intentionally not the same as
+// the public AppModel types in types.ts (e.g. Stage), which describe the
+// *posted* message shape after finalizeStage/appMessage reshape things.
+// ---------------------------------------------------------------------------
+
+                            
+             
+                          
+                            
+                                  
+                                    
+                         
+                             
+    
+           
+                          
+                            
+                                  
+                                    
+                         
+    
+                                           
+                                        
+                            
+                                                                              
+                                                                           
+                                     
+                         
+ 
+
+                         
+             
+               
+                   
+                                                                                                     
+                        
+                              
+                     
+                   
+                        
+ 
+
+                                                                                      
+
+// One accumulated task-attempt record, keyed by `<stageAttemptId>:<index>` (or
+// a unique Symbol when the raw event has no Index). Matches accumulateTask's
+// real built shape (event-handlers.js:169-189).
+                      
+                   
+                  
+                     
+                     
+                        
+                       
+               
+                     
+                          
+                      
+                 
+                     
+                      
+                      
+                       
+                        
+                          
+                          
+                     
+                      
+ 
+
+                       
+             
+               
+                  
+                      
+                      
+                    
+                      
+                           
+                            
+                        
+                             
+                           
+                    
+                          
+                          
+                     
+                      
+                                
+                      
+                                  
+                           
+                                      
+                                    
+                                                        
+                       
+                         
+                             
+                                    
+                                                       
+ 
+
+                              
+             
+                      
+                    
+                         
+                     
+                                  
+                                      
+                                                                         
+                                                                             
+                                                                          
+                             
+ 
+
+                              
+                           
+                                     
+                                               
+                                            
+                                   
+                                       
+                                                 
+                                      
+                                          
+                         
+                                                                              
+                       
+                                               
+                                      
+                                            
+                                 
+ 
+
+export function buildChunkDecoder() {
+  const decoder = new TextDecoder('utf-8');
+  let pending = '';
+  return {
+    decode(buffer            )           {
+      const text = pending + decoder.decode(buffer, { stream: true });
+      const lines = text.split('\n');
+      pending = lines.pop() ?? '';
+      return lines.filter((l) => l.length > 0);
+    },
+    flush()           {
+      const last = pending.trim();
+      pending = '';
+      return last ? [last] : [];
+    },
+  };
+}
+
+export function createState()              {
+  return {
+    app: null,
+    pendingSparkVersion: null,
+    pendingConfig: null,
+    pendingResources: null,
+    stages: new Map(),
+    taskStore: new Map(),
+    sqlExecutions: new Map(),
+    stageToSqlExec: new Map(),
+    sqlExecStages: new Map(),
+    jobs: new Map(),
+    executors: { added: [], removed: [] },
+    skippedLines: 0,
+    accumState: new Map(),
+    rddInfo: new Map(),
+    taskAccumStages: new Map(),
+    evidenceInputs: {
+      environmentUpdates: 0,
+      applicationEnds: 0,
+      stageSubmissions: 0,
+      rddStorageSnapshots: 0,
+      sqlExecutions: 0,
+      resolvedSqlPlans: 0,
+      executorMetricRows: 0,
+      taskRecords: 0,
+    },
+  };
+}
+
+// Normalize the `Spark Properties` payload of SparkListenerEnvironmentUpdate.
+// Modern Spark emits an object of string key→value; older logs use an array of
+// [key, value] pairs. Both collapse to a plain string→string map. The schema
+// (EnvironmentUpdateEventSchema) tolerates raw values that are a bare JSON
+// number or boolean instead of a string, so those are coerced to their
+// string form here, keeping this function's `Record<string, string>`
+// contract intact for every downstream reader of `app.config`.
+                                                    
+export function normalizeSparkProperties(
+  props                                                                                        
+)                         {
+  if (Array.isArray(props)) {
+    const map                         = {};
+    for (const pair of props) {
+      if (Array.isArray(pair) && pair.length >= 2) map[pair[0]] = String(pair[1]);
+    }
+    return map;
+  }
+  const map                         = {};
+  for (const [key, value] of Object.entries(props ?? {})) {
+    map[key] = String(value);
+  }
+  return map;
+}
+
+// Parse a Spark memory-size string to MiB. Spark's JVM-memory configs
+// (spark.{executor,driver}.memory[Overhead]) use bytesConf(ByteUnit.MiB), so a
+// bare number ("10") means MiB. A k/m/g/t suffix sets the unit (an optional
+// trailing "b", as in "gb", is redundant). A lone trailing "b" ("10b") means bytes.
+export function parseSparkMemoryMB(value         )                {
+  if (value == null) return null;
+  const m = String(value).trim().toLowerCase().match(/^([\d.]+)\s*([kmgt]?)(b?)$/);
+  if (!m) return null;
+  const n = parseFloat(m[1]);
+  if (!Number.isFinite(n)) return null;
+  switch (m[2]) {
+    case 'k': return Math.round(n / 1024);
+    case 'g': return Math.round(n * 1024);
+    case 't': return Math.round(n * 1024 * 1024);
+    case 'm': return Math.round(n);
+    default: return m[3] === 'b' ? Math.round(n / (1024 * 1024)) : Math.round(n);
+  }
+}
+
+// Derive an allocated-resource summary (executor/driver memory + cores, plus
+// dynamic-allocation / shuffle-service flags) from the Spark config map.
+// Absent keys degrade to null rather than guessed defaults.
+export function extractResources(config                                           )                   {
+  const cfg = config ?? {};
+  const int = (k        ) => {
+    if (cfg[k] == null) return null;
+    const n = parseInt(cfg[k], 10);
+    return Number.isFinite(n) ? n : null;
+  };
+  const memMB = (k        ) => (cfg[k] != null ? parseSparkMemoryMB(cfg[k]) : null);
+  const bool = (k        ) => (cfg[k] != null ? String(cfg[k]).toLowerCase() === 'true' : null);
+  return {
+    executor: {
+      memory: cfg['spark.executor.memory'] ?? null,
+      memoryMB: memMB('spark.executor.memory'),
+      memoryOverhead: cfg['spark.executor.memoryOverhead'] ?? null,
+      memoryOverheadMB: memMB('spark.executor.memoryOverhead'),
+      cores: int('spark.executor.cores'),
+      instances: int('spark.executor.instances'),
+    },
+    driver: {
+      memory: cfg['spark.driver.memory'] ?? null,
+      memoryMB: memMB('spark.driver.memory'),
+      memoryOverhead: cfg['spark.driver.memoryOverhead'] ?? null,
+      memoryOverheadMB: memMB('spark.driver.memoryOverhead'),
+      cores: int('spark.driver.cores'),
+    },
+    dynamicAllocationEnabled: bool('spark.dynamicAllocation.enabled'),
+    shuffleServiceEnabled: bool('spark.shuffle.service.enabled'),
+    serializer: cfg['spark.serializer'] ?? null,
+  };
+}
+
+// Convert the live rddInfo Map (whose entries carry a mutable `stageIds` Set
+// used for bookkeeping across StageSubmitted events) into a postable copy
+// with `stageIds` as a sorted array, used at every 'app' message post site
+// so the main thread never receives a live Set it can't structured-clone
+// meaningfully or that could keep mutating after being posted.
+function snapshotRddInfo(rddInfo                            )                                     {
+  const out = new Map                               ();
+  for (const [id, r] of rddInfo) {
+    out.set(id, { ...r, stageIds: [...r.stageIds].sort((a, b) => a - b) });
+  }
+  return out;
+}
+
+function snapshotEvidenceInputs(state             )                 {
+  return { ...state.evidenceInputs };
+}
+
+/**
+ * Snapshots the current evidence counters and posts an `app` message. This is
+ * called from `SparkListenerApplicationStart`, `SparkListenerEnvironmentUpdate`,
+ * and `SparkListenerApplicationEnd`, so every streaming `app` message carries a
+ * counter snapshot taken at that moment. Only the terminal snapshot emitted by
+ * `emitParseCompletion` is authoritative; mid-parse snapshots are partial and
+ * must not be used for final evidence-availability conclusions.
+ *
+ * Only ever called once `state.app` has been assigned (from startApplication,
+ * or from updateEnvironment/the ApplicationEnd case which both guard on
+ * `state.app` truthiness before calling this), hence the non-null assertion.
+ */
+function appMessage(state             )                                                 {
+  const evidenceInputs = snapshotEvidenceInputs(state);
+  state.app .evidenceInputs = evidenceInputs;
+  return {
+    type: 'app',
+    data: { ...state.app , rddInfo: snapshotRddInfo(state.rddInfo) },
+  };
+}
+
+export function accumulateTask(event                                    , state             )       {
+  const stageId = event['Stage ID'];
+  const stage = state.stages.get(stageId);
+  if (!stage) return null;
+  // Late TaskEnd for a stage whose StageCompleted already arrived and freed
+  // taskAttempts (finalizeStage), same out-of-order tolerance as
+  // SparkListenerStageExecutorMetrics above; the task's stats are already
+  // baked into the finalized stage and don't need to be re-added.
+  if (stage.taskAttempts === null) return null;
+
+  state.evidenceInputs.taskRecords++;
+
+  const accumulables = event['Task Info']?.Accumulables ?? [];
+  for (const acc of accumulables) {
+    if (!state.taskAccumStages.has(acc.ID)) state.taskAccumStages.set(acc.ID, new Set());
+    state.taskAccumStages.get(acc.ID) .add(stageId);
+  }
+
+  // 'Task Info' itself and its Failed/Killed/Speculative fields are all
+  // optional in the schema (real event-log variants omit them); Partial<>
+  // lets the {} fallback (an absent 'Task Info') type-check while every
+  // read below still defaults via `??`/`||`.
+                                                                                           
+  const info              = event['Task Info'] ?? {};
+  const m = event['Task Metrics'] ?? {};
+  const sr = m['Shuffle Read Metrics'] ?? {};
+  const sw = m['Shuffle Write Metrics'] ?? {};
+  const inp = m['Input Metrics'] ?? {};
+  const out = m['Output Metrics'] ?? {};
+
+  const duration = (info['Finish Time'] ?? 0) - (info['Launch Time'] ?? 0);
+  const failed = !!(info['Failed'] || info['Killed']);
+
+  const record             = {
+    duration, failed,
+    launchTime: info['Launch Time'] ?? 0,
+    finishTime: info['Finish Time'] ?? 0,
+    reason: event['Task End Reason']?.['Reason'] ?? null,
+    speculative: info['Speculative'] === true,
+    host: info['Host'] ?? '',
+    executorId: info['Executor ID'] ?? '',
+    locality: info['Locality'] ?? null,
+    peakExecMem: m['Peak Execution Memory'] ?? 0,
+    gcTime: m['JVM GC Time'] ?? 0,
+    memSpilled: m['Memory Bytes Spilled'] ?? 0,
+    diskSpilled: m['Disk Bytes Spilled'] ?? 0,
+    shuffleRead: (sr['Remote Bytes Read'] ?? 0) + (sr['Local Bytes Read'] ?? 0),
+    shuffleWrite: sw['Shuffle Bytes Written'] ?? 0,
+    fetchWaitTime: sr['Fetch Wait Time'] ?? 0,
+    executorRunTime: m['Executor Run Time'] ?? 0,
+    executorCpuTime: m['Executor CPU Time'] ?? 0,
+    inputBytes: inp['Bytes Read'] ?? 0,
+    outputBytes: out['Bytes Written'] ?? 0,
+  };
+
+  // Dedupe only when Index is present (always true for real Spark event logs).
+  // Without it, every event is treated as a distinct task: preserves legacy
+  // fixture behavior for tests that omit Index.
+  const index = info['Index'];
+  const key                  = index != null ? `${event['Stage Attempt ID'] ?? 0}:${index}` : Symbol('no-index');
+  const existing = stage.taskAttempts.get(key);
+
+  if (!existing) {
+    stage.taskAttempts.set(key, record);
+  } else if (existing.failed && !record.failed) {
+    // A retry succeeded where the earlier attempt failed: the earlier
+    // attempt's time was wasted work, not a metric to fold twice.
+    //
+    // Cause classification: Spark only marks the speculative COPY's own
+    // TaskInfo.Speculative as true, never the original it raced against,
+    // so checking either side of the comparison (not just the discarded
+    // record) is required to catch both "original loses to its speculative
+    // twin" and "speculative twin loses to the original" symmetrically,
+    // regardless of arrival order.
+    if (existing.speculative || record.speculative) {
+      stage.speculationWasteMs += existing.duration;
+      stage.speculationWastedAttempts++;
+    } else {
+      stage.retryWasteMs += existing.duration;
+      stage.wastedAttempts++;
+    }
+    stage.taskAttempts.set(key, record);
+  } else {
+    // Non-winning duplicate (both failed, or a race where a winner is
+    // already recorded): its time is waste, its metrics are discarded.
+    if (existing.speculative || record.speculative) {
+      stage.speculationWasteMs += record.duration;
+      stage.speculationWastedAttempts++;
+    } else {
+      stage.retryWasteMs += record.duration;
+      stage.wastedAttempts++;
+    }
+  }
+
+  return null;
+}
+
+export function resolvePlanTree(
+  rootInfo               ,
+  accumMap                     ,
+  taskAccumStages                          ,
+  executionStageIds                         ,
+)           {
+  const seen = new WeakSet               ();
+  const nodeMap = new WeakMap                         (); // sparkPlanInfoNode → resolved PlanNode
+
+  function makeNode(info               )           {
+    const metrics = (info.metrics ?? []).reduce                                                        ((acc, m) => {
+      if (m.accumulatorId !== undefined && accumMap.has(m.accumulatorId)) {
+        acc.push({ name: m.name, value: accumMap.get(m.accumulatorId) , metricType: m.metricType });
+      }
+      return acc;
+    }, []);
+
+    // Union the stages where this node's accumulator IDs were actually
+    // observed running (Task 2's data), then clip to this SQL execution's own
+    // stage universe (Task 3's data): an accumulator ID can occasionally
+    // belong to a different execution's stages (e.g. a reused subquery),
+    // and without the clip that would misattribute another execution's work.
+    const stageIdSet = new Set        ();
+    for (const m of info.metrics ?? []) {
+      if (m.accumulatorId === undefined) continue;
+      const stages = taskAccumStages.get(m.accumulatorId);
+      if (!stages) continue;
+      for (const sid of stages) {
+        if (executionStageIds && executionStageIds.has(sid)) stageIdSet.add(sid);
+      }
+    }
+
+    const node           = { name: info.nodeName, detail: info.simpleString ?? '', metrics, children: [] };
+    if (stageIdSet.size > 0) node.stageIds = [...stageIdSet].sort((a, b) => a - b);
+    return node;
+  }
+
+  // DFS to collect nodes in discovery order
+  const stack                  = [rootInfo];
+  const order                  = [];
+
+  while (stack.length > 0) {
+    const info = stack.pop() ;
+    if (seen.has(info)) continue;
+    seen.add(info);
+    order.push(info);
+    for (const child of (info.children ?? [])) {
+      if (!seen.has(child)) stack.push(child);
+    }
+  }
+
+  // Build nodes bottom-up (reverse discovery = children before parents)
+  for (let i = order.length - 1; i >= 0; i--) {
+    const info = order[i];
+    const node = makeNode(info);
+    nodeMap.set(info, node);
+    for (const child of (info.children ?? [])) {
+      const childNode = nodeMap.get(child);
+      if (childNode) node.children.push(childNode);
+    }
+  }
+
+  return nodeMap.get(rootInfo) ;
+}
+
+// Maps SparkListenerStageExecutorMetrics' "Executor Metrics" field names
+// (Spark's ExecutorMetricType constants) to our camelCase data-model names.
+// Only listed fields are captured: anything else is ignored (forward-compat
+// with future Spark ExecutorMetricType additions).
+const EXECUTOR_METRIC_FIELD_MAP                         = {
+  JVMHeapMemory: 'jvmHeapMemory', JVMOffHeapMemory: 'jvmOffHeapMemory',
+  OnHeapExecutionMemory: 'onHeapExecutionMemory', OffHeapExecutionMemory: 'offHeapExecutionMemory',
+  OnHeapStorageMemory: 'onHeapStorageMemory', OffHeapStorageMemory: 'offHeapStorageMemory',
+  OnHeapUnifiedMemory: 'onHeapUnifiedMemory', OffHeapUnifiedMemory: 'offHeapUnifiedMemory',
+  DirectPoolMemory: 'directPoolMemory', MappedPoolMemory: 'mappedPoolMemory',
+  ProcessTreeJVMVMemory: 'processTreeJVMVMemory', ProcessTreeJVMRSSMemory: 'processTreeJVMRSSMemory',
+  ProcessTreePythonVMemory: 'processTreePythonVMemory', ProcessTreePythonRSSMemory: 'processTreePythonRSSMemory',
+  ProcessTreeOtherVMemory: 'processTreeOtherVMemory', ProcessTreeOtherRSSMemory: 'processTreeOtherRSSMemory',
+  MinorGCCount: 'minorGCCount', MinorGCTime: 'minorGCTime',
+  MajorGCCount: 'majorGCCount', MajorGCTime: 'majorGCTime', TotalGCTime: 'totalGCTime',
+  ConcurrentGCCount: 'concurrentGCCount', ConcurrentGCTime: 'concurrentGCTime',
+};
+
+export function startApplication(event                                             , state             ) {
+  const config = state.pendingConfig ?? {};
+  state.app = {
+    id: event['App ID'],
+    name: event['App Name'],
+    startTime: event['Timestamp'],
+    endTime: null,
+    sparkVersion: state.pendingSparkVersion ?? event['Spark Version'] ?? null,
+    config,
+    resources: state.pendingResources ?? extractResources(config),
+    rddInfo: state.rddInfo,
+  };
+  return appMessage(state);
+}
+
+export function updateEnvironment(event                                              , state             ) {
+  state.evidenceInputs.environmentUpdates++;
+  const config = normalizeSparkProperties(event['Spark Properties']);
+  const resources = extractResources(config);
+  // EnvironmentUpdate normally precedes ApplicationStart: stash the config
+  // so ApplicationStart can attach it. If it arrives after (a mid-run
+  // update), apply it live and re-post the app so the main thread refreshes.
+  if (state.app) {
+    state.app.config = config;
+    state.app.resources = resources;
+    return appMessage(state);
+  }
+  state.pendingConfig = config;
+  state.pendingResources = resources;
+  return null;
+}
+
+export function startJob(event                                     , state             )       {
+  const stageIds = event['Stage IDs'] ?? [];
+  const sqlExecIdStr = event['Properties']?.['spark.sql.execution.id'];
+  const sqlExecutionId = sqlExecIdStr != null ? parseInt(String(sqlExecIdStr), 10) : null;
+  if (sqlExecutionId != null) {
+    if (!state.sqlExecStages.has(sqlExecutionId)) state.sqlExecStages.set(sqlExecutionId, new Set());
+    const stageSet = state.sqlExecStages.get(sqlExecutionId) ;
+    for (const stageId of stageIds) {
+      state.stageToSqlExec.set(stageId, sqlExecutionId);
+      stageSet.add(stageId);
+    }
+  }
+  const jobId = event['Job ID'];
+  if (jobId != null) {
+    state.jobs.set(jobId, {
+      id: jobId,
+      submissionTime: event['Submission Time'] ?? null,
+      stageIds,
+      sqlExecutionId,
+      result: null, succeeded: null, exception: null, completionTime: null,
+    });
+  }
+  return null;
+}
+
+export function endJob(event                                   , state             )                             {
+  const jobId = event['Job ID'];
+  const result = event['Job Result']?.['Result'] ?? null;
+  const exception = event['Job Result']?.['Exception']?.['Message'] ?? null;
+  const job      = state.jobs.get(jobId) ?? {
+    id: jobId, submissionTime: null, stageIds: [], sqlExecutionId: null,
+    result: null, succeeded: null, exception: null, completionTime: null,
+  };
+  job.result = result;
+  job.succeeded = result === 'JobSucceeded';
+  job.exception = exception;
+  job.completionTime = event['Completion Time'] ?? null;
+  state.jobs.set(jobId, job);
+  return { type: 'job', data: { ...job } };
+}
+
+export function submitStage(event                                           , state             )       {
+  state.evidenceInputs.stageSubmissions++;
+  const info = event['Stage Info'];
+  const id = info['Stage ID'];
+  state.stages.set(id, {
+    id, name: info['Stage Name'] ?? '', details: info['Details'] ?? '',
+    submittedAt: info['Submission Time'] ?? 0, completedAt: 0,
+    taskCount: 0, failedTasks: 0,
+    shuffleReadBytes: 0, shuffleWriteBytes: 0, fetchWaitTime: 0,
+    memoryBytesSpilled: 0, diskBytesSpilled: 0,
+    jvmGCTime: 0, executorRunTime: 0, executorCpuTime: 0,
+    inputBytes: 0, outputBytes: 0,
+    sqlExecutionId: state.stageToSqlExec.get(id) ?? null,
+    parentIds: info['Parent IDs'] ?? [],
+    hostStats: new Map(),
+    speculativeTasks: 0,
+    failureReasons: new Map(),
+    stageFailureReason: null,
+    taskAttempts: new Map(),
+    retryWasteMs: 0,
+    wastedAttempts: 0,
+    speculationWasteMs: 0,
+    speculationWastedAttempts: 0,
+    executorMetrics: new Map(),
+  });
+  mergeStageRddInfo(info, id, state);
+  return null;
+}
+
+export function mergeStageRddInfo(
+  info                                                         ,
+  id        ,
+  state             
+)       {
+  for (const rdd of (info['RDD Info'] ?? [])) {
+    const rddId = rdd['RDD ID'];
+    if (rddId == null) continue;
+    state.evidenceInputs.rddStorageSnapshots++;
+    const sl = rdd['Storage Level'] ?? {};
+    const prev = state.rddInfo.get(rddId);
+    const stageIds = prev?.stageIds ?? new Set        ();
+    stageIds.add(id);
+    state.rddInfo.set(rddId, {
+      id: rddId,
+      name: rdd['Name'] ?? '',
+      callsite: rdd['Callsite'] ?? '',
+      storageLevel: {
+        useDisk: sl['Use Disk'] ?? false,
+        useMemory: sl['Use Memory'] ?? false,
+        deserialized: sl['Deserialized'] ?? false,
+        replication: sl['Replication'] ?? 1,
+      },
+      numPartitions: rdd['Number of Partitions'] ?? 0,
+      // Merge forward, don't overwrite: an RDD being cached for the first
+      // time in THIS stage legitimately reports 0 here (Spark's snapshot
+      // reflects BlockManager state at submission time). If a later
+      // resubmission arrives before the cache numbers are re-observed,
+      // keep the last real value instead of regressing to 0/em-dash.
+      numCachedPartitions: rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
+      memorySize: rdd['Memory Size'] || prev?.memorySize || 0,
+      diskSize: rdd['Disk Size'] || prev?.diskSize || 0,
+      stageIds,
+    });
+  }
+}
+
+// Spark's EventLoggingListener logs these AFTER the stage's
+// SparkListenerStageCompleted, so finalizeStage has already posted
+// (structured-cloned) the stage with an empty executorMetrics Map by the
+// time they arrive. We keep accumulating into the worker-side stage here,
+// then re-post all populated maps once via `stageExecutorMetrics` just
+// before `done` (see collectStageExecutorMetrics) so the main-thread
+// stages are patched before analyze() runs.
+export function recordStageExecutorMetrics(event                                                 , state             )       {
+  const stage = state.stages.get(event['Stage ID']);
+  if (!stage) return null;
+  const raw = event['Executor Metrics'] ?? {};
+  const metrics                         = {};
+  for (const [sparkName, ourName] of Object.entries(EXECUTOR_METRIC_FIELD_MAP)) {
+    if (raw[sparkName] != null) metrics[ourName] = raw[sparkName];
+  }
+  if (Object.keys(metrics).length > 0) {
+    state.evidenceInputs.executorMetricRows++;
+  }
+  stage.executorMetrics.set(event['Executor ID'], metrics);
+  return null;
+}
+
+export function startSqlExecution(event                                              , state             )                                            {
+  state.evidenceInputs.sqlExecutions++;
+  const sparkPlanInfo = event.sparkPlanInfo ?? null;
+  const exec                     = {
+    id: event.executionId, description: event.description ?? '',
+    startTime: event.time, endTime: null, stageIds: [],
+    physicalPlanDescription: event.physicalPlanDescription ?? '',
+    sparkPlanInfo,
+    hadAdaptiveUpdate: false,
+  };
+  state.sqlExecutions.set(exec.id, exec);
+  if (sparkPlanInfo !== null) {
+    state.accumState.set(exec.id, new Map());
+  }
+  return { type: 'sql', data: exec };
+}
+
+export function endSqlExecution(
+  event                                            ,
+  state             
+)                                                                                                                            {
+  const exec = state.sqlExecutions.get(event.executionId);
+  if (exec) exec.endTime = event.time;
+
+  const planInfo = exec?.sparkPlanInfo ?? null;
+  if (!planInfo || !planInfo.nodeName) {
+    state.accumState.delete(event.executionId);
+    return exec ? { type: 'sql', data: { ...exec } } : null;
+  }
+
+  const accumMap = state.accumState.get(event.executionId) ?? new Map                ();
+  const planTree = resolvePlanTree(planInfo, accumMap, state.taskAccumStages, state.sqlExecStages.get(event.executionId));
+  state.accumState.delete(event.executionId);
+
+  state.evidenceInputs.resolvedSqlPlans++;
+  return { type: 'sqlPlan', data: { executionId: event.executionId, planTree } };
+}
+
+export function applyDriverAccumUpdates(event                                               , state             )       {
+  const { executionId, accumUpdates } = event;
+  if (!state.accumState.has(executionId)) return null; // late update, discard
+  const map = state.accumState.get(executionId) ;
+  for (const [accId, delta] of accumUpdates) {
+    map.set(accId, (map.get(accId) ?? 0) + delta);
+  }
+  return null;
+}
+
+export function applyAdaptiveExecutionUpdate(
+  event                                                       ,
+  state             ,
+)                                                   {
+  const exec = state.sqlExecutions.get(event.executionId);
+  if (!exec) return null; // late update for an unseen execution, discard (same pattern as applyDriverAccumUpdates)
+  if (event.sparkPlanInfo != null) exec.sparkPlanInfo = event.sparkPlanInfo;
+  if (event.physicalPlanDescription != null) exec.physicalPlanDescription = event.physicalPlanDescription;
+  exec.hadAdaptiveUpdate = true;
+  // Re-emit a 'sql' message (same shape startSqlExecution posts) so the
+  // browser's structured-cloned appModel.sql copy actually sees the flip:
+  // without this, hadAdaptiveUpdate only ever reads true via collectRun's
+  // Node-path object aliasing (no postMessage clone in that path), never in
+  // the shipping worker. Shallow copy so the posted object isn't the same
+  // mutable reference the worker keeps mutating (mirrors endSqlExecution's
+  // `{ ...exec }` pattern above).
+  return { type: 'sql', data: { ...exec } };
+}
+
+export function addExecutor(event                                          , state             )                                                 {
+  const ev                     = {
+    kind: 'added', timestamp: event['Timestamp'],
+    executorId: event['Executor ID'],
+    host: event['Executor Info']?.['Host'] ?? '',
+    totalCores: event['Executor Info']?.['Total Cores'] ?? 0,
+    // Resource Profile Id lives inside Executor Info in real Spark logs;
+    // fall back to a top-level key for any variant that hoists it.
+    resourceProfileId: event['Executor Info']?.['Resource Profile Id'] ?? event['Resource Profile Id'] ?? null,
+  };
+  state.executors.added.push(ev);
+  return { type: 'executor', data: ev };
+}
+
+export function removeExecutor(event                                            , state             )                                                   {
+  const ev                       = {
+    kind: 'removed', timestamp: event['Timestamp'],
+    executorId: event['Executor ID'],
+    reason: event['Removed Reason'] ?? '',
+  };
+  state.executors.removed.push(ev);
+  return { type: 'executor', data: ev };
+}
+
+export function processEvent(event            , state             )          {
+  switch (event.Event) {
+    case 'SparkListenerLogStart':
+      state.pendingSparkVersion = event['Spark Version'] ?? null;
+      return null;
+
+    case 'SparkListenerApplicationStart':
+      return startApplication(event, state);
+
+    case 'SparkListenerEnvironmentUpdate':
+      return updateEnvironment(event, state);
+
+    case 'SparkListenerApplicationEnd':
+      state.evidenceInputs.applicationEnds++;
+      if (state.app) {
+        state.app.endTime = event['Timestamp'];
+        return appMessage(state);
+      }
+      return null;
+
+    case 'SparkListenerJobStart':
+      return startJob(event, state);
+
+    case 'SparkListenerJobEnd':
+      return endJob(event, state);
+
+    case 'SparkListenerStageSubmitted':
+      return submitStage(event, state);
+
+    case 'SparkListenerStageCompleted': {
+      const info = event['Stage Info'];
+      const id = info['Stage ID'];
+      const stage = state.stages.get(id);
+      if (!stage) return null;
+      stage.completedAt = info['Completion Time'] ?? 0;
+      stage.stageFailureReason = info['Failure Reason'] ?? null;
+      // finalizeStage (Task 5, stage-quantiles.ts) deliberately keeps its
+      // `stage` parameter typed as a loose Record (see that module's own
+      // comment); StageRecord's real shape (more precise than that loose
+      // type, e.g. taskAttempts values carry string/boolean fields too, not
+      // just numbers) is bridged across that boundary with an explicit cast
+      // rather than re-litigating finalizeStage's own established signature.
+      return finalizeStage(
+        id,
+        stage                                                                                                      ,
+        state
+      );
+    }
+
+    case 'SparkListenerStageExecutorMetrics':
+      return recordStageExecutorMetrics(event, state);
+
+    case 'SparkListenerTaskEnd':
+      return accumulateTask(event, state);
+
+    case 'org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart':
+      return startSqlExecution(event, state);
+
+    case 'org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate':
+      return applyAdaptiveExecutionUpdate(event, state);
+
+    case 'org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd':
+      return endSqlExecution(event, state);
+
+    case 'org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates':
+      return applyDriverAccumUpdates(event, state);
+
+    case 'SparkListenerExecutorAdded':
+      return addExecutor(event, state);
+
+    case 'SparkListenerExecutorRemoved':
+      return removeExecutor(event, state);
+
+    default:
+      return assertNever(event);
+  }
+}
+
+// Derived (not hand-maintained) from SparkEventSchema itself, so this set
+// can never drift from the discriminated union's real literal list.
+const KNOWN_EVENT_TYPES                      = new Set(
+  SparkEventSchema.options.map((option) => option.shape.Event.value)
+);
+
+export function dispatchLine(line        , state             , emit                        )       {
+  let parsed         ;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    state.skippedLines++;
+    return;
+  }
+  // A real Spark event log carries many event types this tool has never
+  // modeled (BlockManagerAdded, TaskStart, ExecutorMetricsUpdate, ...): that
+  // was always true pre-migration and is completely benign, so an `Event`
+  // value outside our 15 modeled literals is silently ignored exactly like
+  // before, never counted toward skippedLines. Only a value that IS one of
+  // the 15 known types but fails ITS schema is a genuine, valuable signal
+  // worth surfacing as a skipped line.
+  const eventType = (parsed                              )?.Event;
+  if (typeof eventType !== 'string' || !KNOWN_EVENT_TYPES.has(eventType)) {
+    return;
+  }
+  const result = SparkEventSchema.safeParse(parsed);
+  if (!result.success) {
+    state.skippedLines++;
+    return;
+  }
+  let msg         ;
+  try {
+    msg = processEvent(result.data, state);
+  } catch (err) {
+    // processEvent's switch is exhaustive over the schema-validated union
+    // (assertNever in the default case), so this should be unreachable for
+    // any event that made it this far. If it ever fires, it's a genuine bug
+    // in a handler, not malformed input; log it distinctly so it doesn't
+    // masquerade as an ordinary skipped-line (malformed/invalid data) count.
+    console.error('processEvent threw for an already-validated event:', err);
+    state.skippedLines++;
+    return;
+  }
+  if (msg) emit(msg);
+}
+
+// Gather every stage's post-completion-populated executorMetrics for a single
+// re-post just before `done` (see the SparkListenerStageExecutorMetrics case
+// for why the per-stage message posted at completion is empty). Returns a
+// Map<stageId, Map<execId, metrics>>; empty when the log had no
+// spark.eventLog.logStageExecutorMetrics data.
+export function collectStageExecutorMetrics(state             )                                                   {
+  const out = new Map                                             ();
+  for (const [id, stage] of state.stages) {
+    if (stage.executorMetrics instanceof Map && stage.executorMetrics.size > 0) {
+      out.set(id, stage.executorMetrics);
+    }
+  }
+  return out;
+}
+
+export function emitParseCompletion(state             , emit                        , linesProcessed        )       {
+  emit({ type: 'progress', pct: 1, linesProcessed });
+  emit({ type: 'runAggregates', data: computeRunAggregates(state.taskStore) });
+  emit({ type: 'stageExecutorMetrics', data: collectStageExecutorMetrics(state) });
+  emit(appMessage(state));
+  emit({ type: 'done', skippedLines: state.skippedLines });
+  state.accumState.clear();
+}