sparkforensics-mcp 0.3.0 → 0.4.0

package/vendor-core/docs-config.js

@@ -1,5 +1,3 @@
-import { detectorCatalog } from './detectors.js';
-
 // Single source of the docs-panel URL surface. DOCS_BASE_DIR is the built docs-site path that
 // serves the tuning reference, relative to the app's origin.
 export const DOCS_BASE_DIR         = 'docs/tuning-reference';
@@ -90,40 +88,6 @@ export function isKnownDocAnchor(anchor         )          {
   return KNOWN_DOC_ANCHORS.has(String(anchor));
 }
 
-// Finding types that never appear as their own DETECTORS entry's type: the parent entry declares
-// a different type because one plan-walk covers two rules (see broadcastSizing). Map to the parent.
-const TYPE_ALIASES                         = {
-  underBroadcast: 'broadcastSizing',
-  overBroadcast: 'broadcastSizing',
-};
-
-// DETECTORS is static, so this grouping is built once (lazily) instead of re-scanning per
-// docAnchorForType call (called once per TagBadge per render).
-let anchorsByTypeCache                                                  ;
-
-function anchorsByType()                                       {
-  if (!anchorsByTypeCache) {
-    anchorsByTypeCache = new Map();
-    for (const entry of detectorCatalog()) {
-      const anchors = anchorsByTypeCache.get(entry.type) ?? new Set();
-      anchors.add(entry.docAnchor);
-      anchorsByTypeCache.set(entry.type, anchors);
-    }
-  }
-  return anchorsByTypeCache;
-}
-
-/** Resolves a finding `type` to its documented anchor from detectorCatalog(). Returns undefined
- * when entries sharing the type disagree on docAnchor (only configAudit today), or when the
- * resolved anchor isn't in the allowlist (isKnownDocAnchor, the same gate DocsLink uses). */
-export function docAnchorForType(type        )                     {
-  const resolvedType = TYPE_ALIASES[type] ?? type;
-  const anchors = anchorsByType().get(resolvedType) ?? new Set();
-  if (anchors.size !== 1) return undefined;
-  const [anchor] = anchors;
-  return anchor && isKnownDocAnchor(anchor) ? anchor : undefined;
-}
-
 /** The docAnchor every finding in `findings` carries, or undefined when they disagree or any lacks
  * one. For a badge standing for several findings (a widget header, a grouped row) whose type-level
  * docAnchorForType can't pick one: configAudit's sub-checks each stamp their own anchor. */

package/vendor-core/docs-content/chapters/nav-index.json

@@ -3,6 +3,7 @@
     "anchor": "intro",
     "section": "Getting Started",
     "title": "Introduction",
+    "brief": "Guide overview: what the reference covers, the severity-dot legend, and the tag system linking into the Bottleneck Reference.",
     "keywords": [
       "severity-dot",
       "tag-system"
@@ -14,6 +15,7 @@
     "anchor": "spark-architecture",
     "section": "Optimization Guide",
     "title": "Spark Execution Model",
+    "brief": "Spark's execution model: Driver/Executor split, DAGScheduler/TaskScheduler roles, stage boundaries at shuffles, and pipelining of narrow transformations.",
     "keywords": [
       "driver-executor",
       "dagscheduler",
@@ -27,6 +29,7 @@
     "anchor": "memory-model",
     "section": "Optimization Guide",
     "title": "Memory Management",
+    "brief": "Unified memory management: on-heap/off-heap execution vs. storage regions, spill triggers, and executor memory overhead sizing.",
     "keywords": [
       "unified-memory",
       "execution-memory",
@@ -41,6 +44,7 @@
     "anchor": "partitioning",
     "section": "Optimization Guide",
     "title": "Partitioning",
+    "brief": "Partition sizing and reshaping: repartition vs. coalesce mechanics, shuffle-partition tuning, and skew detection thresholds.",
     "keywords": [
       "repartition",
       "coalesce",
@@ -53,6 +57,7 @@
     "anchor": "joins",
     "section": "Optimization Guide",
     "title": "Join Optimization",
+    "brief": "Physical join strategy selection: broadcast vs. shuffle joins, join hints, bucketing, and AQE's runtime join-strategy switching.",
     "keywords": [
       "broadcast",
       "shuffle-join",
@@ -66,6 +71,7 @@
     "anchor": "shuffle",
     "section": "Optimization Guide",
     "title": "Shuffle",
+    "brief": "Shuffle internals: SortShuffleManager mechanics, shuffle-avoidance paths (bucketing, Storage Partition Join), and shuffle-tuning configs.",
     "keywords": [
       "sortshufflemanager",
       "bucketing",
@@ -78,6 +84,7 @@
     "anchor": "data-formats",
     "section": "Optimization Guide",
     "title": "Data Formats",
+    "brief": "Columnar file format tradeoffs: Parquet vs. ORC internals, splittability, predicate pushdown, and compression codec choice.",
     "keywords": [
       "parquet",
       "orc",
@@ -91,6 +98,7 @@
     "anchor": "table-formats",
     "section": "Optimization Guide",
     "title": "Table Formats",
+    "brief": "Lakehouse table-format tuning across Delta Lake, Iceberg, and Hudi: file sizing, compaction/OPTIMIZE, clustering (Z-order/liquid/sort), metadata and manifest overhead, snapshot/version expiry, and deletion vectors vs. merge-on-read/copy-on-write.",
     "keywords": [
       "delta",
       "iceberg",
@@ -107,6 +115,7 @@
     "anchor": "caching",
     "section": "Optimization Guide",
     "title": "Caching & Persistence",
+    "brief": "cache()/persist() semantics, storage levels, eviction behavior, and checkpointing as a lineage-truncation alternative.",
     "keywords": [
       "cache",
       "persist",
@@ -120,6 +129,7 @@
     "anchor": "pyspark",
     "section": "Optimization Guide",
     "title": "PySpark Specifics",
+    "brief": "PySpark-specific performance: UDF serialization cost, Arrow-optimized and pandas UDF types, and Python-worker memory configs.",
     "keywords": [
       "udf",
       "arrow",
@@ -133,6 +143,7 @@
     "anchor": "aqe",
     "section": "Optimization Guide",
     "title": "Adaptive Query Execution",
+    "brief": "AQE's runtime re-optimization loop: post-shuffle partition coalescing, join-strategy promotion, and skew-partition splitting.",
     "keywords": [
       "partition-coalescing",
       "join-strategy-promotion",
@@ -145,6 +156,7 @@
     "anchor": "cluster-config",
     "section": "Optimization Guide",
     "title": "Cluster Tuning",
+    "brief": "Cluster-level sizing: executor core/memory formulas, container memory budgeting, dynamic allocation, and locality-wait tuning.",
     "keywords": [
       "executor-sizing",
       "container-memory",
@@ -158,6 +170,7 @@
     "anchor": "anti-patterns",
     "section": "Optimization Guide",
     "title": "Anti-Patterns",
+    "brief": "Checklist of a dozen recurring Spark performance anti-patterns, each with what it is, how to detect it, why it hurts, and how to fix it.",
     "keywords": [
       "checklist",
       "anti-pattern"
@@ -169,6 +182,7 @@
     "anchor": "bottleneck-skew",
     "section": "Detector Catalog",
     "title": "Task Skew",
+    "brief": "Detecting and mitigating task skew: oversized shuffle partitions vs. the median, AQE skew-join splitting, and manual salting as the pre-AQE fallback.",
     "keywords": [
       "skew",
       "aqe-skew-join",
@@ -181,6 +195,7 @@
     "anchor": "bottleneck-shuffle",
     "section": "Detector Catalog",
     "title": "Shuffle I/O",
+    "brief": "Detecting and mitigating shuffle I/O bottlenecks: read/write byte thresholds, fetch-wait time, and shuffle-avoidance and tuning levers.",
     "keywords": [
       "shuffle-bytes",
       "fetch-wait"
@@ -192,6 +207,7 @@
     "anchor": "bottleneck-spill",
     "section": "Detector Catalog",
     "title": "Memory / Disk Spill",
+    "brief": "Detecting and mitigating memory/disk spill: TaskMemoryManager pressure, the skew-vs-volume spill classification, and matching the fix to the cause.",
     "keywords": [
       "spill",
       "taskmemorymanager"
@@ -203,6 +219,7 @@
     "anchor": "bottleneck-gc",
     "section": "Detector Catalog",
     "title": "GC Pressure",
+    "brief": "Detecting and mitigating GC pressure: the gcPct warning/critical thresholds, oversized executors and memory.fraction as causes, and off-heap/collector fixes.",
     "keywords": [
       "gc-pressure",
       "memory-fraction",
@@ -215,6 +232,7 @@
     "anchor": "bottleneck-cold-start",
     "section": "Detector Catalog",
     "title": "Cold Start",
+    "brief": "Detecting and mitigating cold start: the firstStageSubmittedAt gap before the first stage is submitted, and executor-sizing/dynamic-allocation readiness fixes.",
     "keywords": [
       "cold-start",
       "dynamic-allocation"
@@ -226,6 +244,7 @@
     "anchor": "bottleneck-utilization",
     "section": "Detector Catalog",
     "title": "Executor Utilization",
+    "brief": "Detecting and mitigating low executor utilization: the avg/peak active-executor ratio, under-partitioning and coalesce-caused parallelism loss, and dynamic-allocation over-provisioning.",
     "keywords": [
       "executor-utilization",
       "under-partitioning",
@@ -238,6 +257,7 @@
     "anchor": "bottleneck-slow-host",
     "section": "Detector Catalog",
     "title": "Slow Host",
+    "brief": "Detecting and mitigating a slow host: host-wide task duration inflation versus the cluster median, and speculative-execution or locality-wait fixes.",
     "keywords": [
       "slow-host",
       "speculative-execution"
@@ -249,6 +269,7 @@
     "anchor": "bottleneck-failures",
     "section": "Detector Catalog",
     "title": "Task Failures",
+    "brief": "Detecting and mitigating task failures: the TaskEndReason taxonomy, failed-task-share thresholds, and distinguishing memory-driven ExecutorLostFailure from application-code ExceptionFailure.",
     "keywords": [
       "taskendreason",
       "executorlostfailure",
@@ -261,6 +282,7 @@
     "anchor": "bottleneck-straggler",
     "section": "Detector Catalog",
     "title": "Stragglers",
+    "brief": "Detecting and mitigating straggler tasks: the 4x-median-duration rule, distinguishing stragglers from GC pauses or skew, and speculative-execution tuning.",
     "keywords": [
       "straggler",
       "speculative-execution"
@@ -272,6 +294,7 @@
     "anchor": "bottleneck-retry-waste",
     "section": "Detector Catalog",
     "title": "Retry Waste",
+    "brief": "Detecting and mitigating wasted retry compute: executorRunTime burned by superseded task attempts, and distinguishing executor-loss from fetch-failure causes.",
     "keywords": [
       "retry-waste",
       "fetch-failure"
@@ -283,6 +306,7 @@
     "anchor": "bottleneck-tiny-tasks",
     "section": "Detector Catalog",
     "title": "Tiny Tasks",
+    "brief": "Detecting and mitigating tiny-task overhead: scheduling/serialization cost dominating once partitions shrink too far, and coalesce/repartition sizing fixes.",
     "keywords": [
       "tiny-tasks",
       "scheduling-overhead"
@@ -294,6 +318,7 @@
     "anchor": "bottleneck-job-failure-rate",
     "section": "Detector Catalog",
     "title": "Job Failure Rate",
+    "brief": "Detecting and mitigating job-level failure: the task/stage/executor retry-budget escalation ladder that turns isolated failures into an aborted job.",
     "keywords": [
       "job-failure-rate",
       "retry-budget"
@@ -305,6 +330,7 @@
     "anchor": "bottleneck-memory-utilization",
     "section": "Detector Catalog",
     "title": "Memory Utilization",
+    "brief": "Detecting and mitigating held-but-idle executor memory: cores sitting idle below allocated slots, dynamic-allocation idle timeouts that reclaim an executor's memory band, and the low-confidence peak-vs-allocated waste estimate.",
     "keywords": [
       "memory-utilization",
       "idle-timeout"
@@ -316,6 +342,7 @@
     "anchor": "bottleneck-duplicate-plan-subtree",
     "section": "Detector Catalog",
     "title": "Duplicate Plan Subtree",
+    "brief": "Detecting a repeated scan or exchange subtree in a physical plan, how exchange reuse and cache()/persist() avoid recomputing it, and why inferring duplication from the plan is a medium-confidence signal.",
     "keywords": [
       "duplicate-plan-subtree",
       "exchange-reuse"
@@ -327,6 +354,7 @@
     "anchor": "bottleneck-small-files",
     "section": "Detector Catalog",
     "title": "Small Files",
+    "brief": "Detecting and mitigating the small-files problem on read and write: metadata and I/O overhead of many tiny files, and coalesce()/repartition()/compaction and AQE advisoryPartitionSizeInBytes as output-size controls.",
     "keywords": [
       "small-files",
       "advisorypartitionsizeinbytes"
@@ -338,6 +366,7 @@
     "anchor": "bottleneck-broadcast-sizing",
     "section": "Detector Catalog",
     "title": "Broadcast Sizing",
+    "brief": "Detecting under- and over-broadcast joins: spark.sql.autoBroadcastJoinThreshold and its default, when to broadcast the smaller side versus shuffle, and the driver-collection and executor-memory failure modes of broadcasting too large a table.",
     "keywords": [
       "broadcast-threshold",
       "driver-collection"
@@ -349,6 +378,7 @@
     "anchor": "metrics",
     "section": "Reference",
     "title": "Metrics Glossary",
+    "brief": "Glossary of every metric surfaced elsewhere on the site: what each one measures, its event-log source field, and what a problematic value looks like.",
     "keywords": [
       "metrics-glossary",
       "event-log"
@@ -360,6 +390,7 @@
     "anchor": "config",
     "section": "Reference",
     "title": "Spark Config Quick-Reference",
+    "brief": "Cross-reference of the Spark/PySpark config properties and JVM GC flags discussed elsewhere in the guide, with defaults and version notes.",
     "keywords": [
       "spark-config",
       "gc-flags"

package/vendor-core/docs-content/detection/cstor.md

@@ -2,3 +2,12 @@
 
 A persisted dataset is not fully cached in memory, or is spilling to disk.
 Raise executor memory, or shrink the cached dataset.
+
+The cached-partition counts and sizes come from `SparkListenerBlockUpdated`
+events, which Spark writes only when
+`spark.eventLog.logBlockUpdates.enabled=true`. Since Spark 2.3 the RDD
+storage figures in stage-submission events are always 0, and they are
+used only as a fallback. When a Spark 2.3+ run persists RDDs but its log has
+neither and block-update logging was off, the check reports that the cache could not be
+checked instead of passing it: turn `spark.eventLog.logBlockUpdates.enabled`
+on and rerun to measure eviction and disk spillover.

package/vendor-core/docs-site-config.js

@@ -14,3 +14,6 @@ import { typeTag } from './format-utils.js';
 export function findingGuideUrl(type        )         {
   return `docs/user-guide/understanding-findings.html#${typeTag(type).toLowerCase()}`;
 }
+
+// Every other way to get a log (cloud consoles, bastions, copying from storage).
+export const ALTERNATIVE_LOG_RETRIEVAL_URL = 'docs/user-guide/alternative-log-retrieval.html';

package/vendor-core/event-handlers.js

@@ -14,6 +14,7 @@ import {
   DriverAccumUpdatesEventSchema,
   ExecutorAddedEventSchema,
   ExecutorRemovedEventSchema,
+  BlockUpdatedEventSchema,
                   
                      
 } from './event-schemas.js';
@@ -62,7 +63,22 @@ import { computeRunAggregates } from './run-aggregates.js';
                               
                      
                    
+                                                                                                    
+                                                                                                          
+                                                                                              
+                                            
                         
+ 
+
+// Live per-RDD block residency rebuilt from SparkListenerBlockUpdated. Keyed by partition and
+// executor because a block's status is per BlockManager: replicas and re-caches on another
+// executor are separate entries, as in Spark's own AppStatusListener.
+                         
+                                                                                   
+                                           
+                     
+                   
+                               
  
 
                                                                                       
@@ -147,6 +163,12 @@ const MAX_TASK_SAMPLES = 20;
                          
                              
                                     
+                                                                                                
+                                                                                                          
+                                           
+                                                                                             
+                                               
+                                
                                                        
  
 
@@ -178,6 +200,10 @@ const MAX_TASK_SAMPLES = 20;
                        
                                                
                                       
+                                        
+                                                                                                 
+                                                                      
+                          
                                             
                                  
                                                                                      
@@ -367,6 +393,8 @@ export function createState()              {
     skippedLines: 0,
     accumState: new Map(),
     rddInfo: new Map(),
+    rddBlocks: new Map(),
+    rddBlockUpdates: 0,
     taskAccumStages: new Map(),
     pendingAdaptiveUpdates: new Map(),
     resolvedPlanExecutions: new Set(),
@@ -478,6 +506,7 @@ function snapshotEvidenceInputs(state             )                 {
 function appMessage(state             )                                                 {
   const evidenceInputs = snapshotEvidenceInputs(state);
   state.app .evidenceInputs = evidenceInputs;
+  state.app .rddBlockUpdates = state.rddBlockUpdates;
   return {
     type: 'app',
     data: { ...state.app , rddInfo: snapshotRddInfo(state.rddInfo) },
@@ -509,8 +538,12 @@ export function accumulateTask(event                                    , state
   const stage = state.stages.get(stageId);
   if (!stage) return null;
   // Late TaskEnd for a stage whose StageCompleted already freed taskAttempts (finalizeStage): its
-  // stats are already baked into the finalized stage, don't re-add.
-  if (stage.taskAttempts === null) return null;
+  // stats are already baked into the finalized stage, don't re-add. The one exception is a losing
+  // speculative attempt, whose wasted time the finalized stage never saw.
+  if (stage.taskAttempts === null) {
+    accountLateSpeculativeLoser(event, stage);
+    return null;
+  }
 
   state.evidenceInputs.taskRecords++;
 
@@ -566,6 +599,7 @@ export function accumulateTask(event                                    , state
 
   if (!existing) {
     stage.taskAttempts.set(key, record);
+    if (record.speculative && !record.failed) stage.speculativeWinners.add(key);
   } else if (existing.failed && !record.failed) {
     // A retry succeeded where the earlier attempt failed: the earlier attempt's time was wasted.
     // Spark marks only the speculative COPY's Speculative flag, never the original it raced, so
@@ -581,6 +615,7 @@ export function accumulateTask(event                                    , state
       }
     }
     stage.taskAttempts.set(key, record);
+    if (record.speculative) stage.speculativeWinners.add(key);
   } else {
     // Non-winning duplicate (both failed, or a race where a winner is
     // already recorded): its time is waste, its metrics are discarded.
@@ -599,6 +634,21 @@ export function accumulateTask(event                                    , state
   return null;
 }
 
+// Spark kills the losing copy of a speculative race only once the stage finishes ("Stage
+// cancelled: Stage finished"), so that loser's TaskEnd normally lands after StageCompleted. Count
+// its time as speculation waste, pairing it the same way accumulateTask does: the late attempt is
+// the speculative copy itself, or the original that a speculative winner beat. Every other stat
+// of a late attempt stays excluded, as the finalized stage already posted them.
+function accountLateSpeculativeLoser(event                                    , stage             )       {
+  const info = event['Task Info'];
+  if (info?.['Index'] == null) return;
+  const key = `${event['Stage Attempt ID'] ?? 0}:${info['Index']}`;
+  if (info['Speculative'] !== true && !stage.speculativeWinners.has(key)) return;
+  stage.speculationWasteMs += (info['Finish Time'] ?? 0) - (info['Launch Time'] ?? 0);
+  stage.speculationWastedAttempts++;
+  stage.lateSpeculationWaste = true;
+}
+
 export function resolvePlanTree(
   rootInfo               ,
   accumMap                     ,
@@ -828,6 +878,8 @@ export function submitStage(event                                           , st
     wastedAttempts: 0,
     speculationWasteMs: 0,
     speculationWastedAttempts: 0,
+    speculativeWinners: new Set(),
+    lateSpeculationWaste: false,
     executorMetrics: new Map(),
   });
   mergeStageRddInfo(info, id, state);
@@ -847,11 +899,16 @@ export function mergeStageRddInfo(
     const prev = state.rddInfo.get(rddId);
     const stageIds = prev?.stageIds ?? new Set        ();
     stageIds.add(id);
+    // Block updates are the authoritative source once seen: never let a later RDD Info snapshot
+    // (0 on Spark 2.3+) replace them, nor the NONE level an unpersist() leaves on ancestor RDDs
+    // listed by later stages.
+    const fromBlocks = prev?.storageSource === 'blockUpdates';
+    const persisted = Boolean(sl['Use Disk'] || sl['Use Memory']);
     state.rddInfo.set(rddId, {
       id: rddId,
       name: rdd['Name'] ?? '',
       callsite: rdd['Callsite'] ?? '',
-      storageLevel: {
+      storageLevel: fromBlocks && !persisted ? prev.storageLevel : {
         useDisk: sl['Use Disk'] ?? false,
         useMemory: sl['Use Memory'] ?? false,
         deserialized: sl['Deserialized'] ?? false,
@@ -861,14 +918,88 @@ export function mergeStageRddInfo(
       // Merge forward, don't overwrite: an RDD cached for the first time in THIS stage legitimately
       // reports 0 (snapshot reflects BlockManager state at submission). Keep the last real value on
       // a resubmission instead of regressing to 0.
-      numCachedPartitions: rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
-      memorySize: rdd['Memory Size'] || prev?.memorySize || 0,
-      diskSize: rdd['Disk Size'] || prev?.diskSize || 0,
+      numCachedPartitions: fromBlocks ? prev.numCachedPartitions : rdd['Number of Cached Partitions'] || prev?.numCachedPartitions || 0,
+      memorySize: fromBlocks ? prev.memorySize : rdd['Memory Size'] || prev?.memorySize || 0,
+      diskSize: fromBlocks ? prev.diskSize : rdd['Disk Size'] || prev?.diskSize || 0,
+      storageSource: prev?.storageSource ?? 'rddInfo',
       stageIds,
     });
   }
 }
 
+const RDD_BLOCK_ID = /^rdd_(\d+)_(\d+)$/;
+
+function dropRddBlock(rdd               , key        )       {
+  const prev = rdd.blocks.get(key);
+  if (!prev) return;
+  rdd.memorySize -= prev.memorySize;
+  rdd.diskSize -= prev.diskSize;
+  const replicas = (rdd.replicasByPartition.get(prev.partition) ?? 1) - 1;
+  if (replicas > 0) rdd.replicasByPartition.set(prev.partition, replicas);
+  else rdd.replicasByPartition.delete(prev.partition);
+  rdd.blocks.delete(key);
+}
+
+/**
+ * Folds one SparkListenerBlockUpdated into its RDD's live residency, then publishes the RDD's
+ * peak state to rddInfo: the most partitions resident at once, with the memory/disk bytes at the
+ * latest moment that peak held. A peak rather than the final state, because an unpersist() (or
+ * the app's own cleanup) removes every block before the log ends, and a final snapshot would
+ * read as "nothing was cached". Ties refresh, so a partition dropping from memory to disk after
+ * the peak still shows up in diskSize. Non-RDD blocks (broadcast, shuffle, task results) are ignored.
+ */
+export function recordBlockUpdate(event                                         , state             )       {
+  const info = event['Block Updated Info'];
+  const match = RDD_BLOCK_ID.exec(info['Block ID']);
+  if (!match) return null;
+  state.rddBlockUpdates++;
+  const rddId = Number(match[1]);
+  const partition = Number(match[2]);
+  const sl = info['Storage Level'] ?? {};
+  // Spark's StorageLevel.isValid: a removal or eviction reports level NONE.
+  const resident = Boolean(sl['Use Memory'] || sl['Use Disk']) && (sl['Replication'] ?? 1) > 0;
+
+  let rdd = state.rddBlocks.get(rddId);
+  if (!rdd) {
+    rdd = { blocks: new Map(), replicasByPartition: new Map(), memorySize: 0, diskSize: 0, peakCachedPartitions: 0 };
+    state.rddBlocks.set(rddId, rdd);
+  }
+  const key = `${partition}@${info['Block Manager ID']?.['Executor ID'] ?? ''}`;
+  dropRddBlock(rdd, key);
+  if (resident) {
+    // Sizes count only where the level says the block lives, as Spark's AppStatusListener does: a
+    // drop from memory to disk reports Use Memory false but still carries the dropped bytes as
+    // Memory Size (BlockManager reports max(memSize, droppedMemorySize)).
+    const block = {
+      partition,
+      memorySize: sl['Use Memory'] ? info['Memory Size'] ?? 0 : 0,
+      diskSize: sl['Use Disk'] ? info['Disk Size'] ?? 0 : 0,
+    };
+    rdd.blocks.set(key, block);
+    rdd.memorySize += block.memorySize;
+    rdd.diskSize += block.diskSize;
+    rdd.replicasByPartition.set(partition, (rdd.replicasByPartition.get(partition) ?? 0) + 1);
+  }
+
+  const record = state.rddInfo.get(rddId) ?? {
+    // A block reported before any stage listed its RDD: name and partition count arrive with
+    // the next StageSubmitted (mergeStageRddInfo keeps the block-derived sizes).
+    id: rddId, name: '', callsite: '',
+    storageLevel: { useDisk: Boolean(sl['Use Disk']), useMemory: Boolean(sl['Use Memory']), deserialized: false, replication: sl['Replication'] ?? 1 },
+    numPartitions: 0, numCachedPartitions: 0, memorySize: 0, diskSize: 0,
+    storageSource: 'blockUpdates'         , stageIds: new Set        (),
+  };
+  record.storageSource = 'blockUpdates';
+  if (rdd.replicasByPartition.size >= rdd.peakCachedPartitions) {
+    rdd.peakCachedPartitions = rdd.replicasByPartition.size;
+    record.numCachedPartitions = rdd.peakCachedPartitions;
+    record.memorySize = rdd.memorySize;
+    record.diskSize = rdd.diskSize;
+  }
+  state.rddInfo.set(rddId, record);
+  return null;
+}
+
 // Spark logs these AFTER SparkListenerStageCompleted, so finalizeStage has already posted the
 // stage with an empty executorMetrics Map. Keep accumulating worker-side, then re-post all maps
 // once via `stageExecutorMetrics` just before `done` so main-thread stages are patched before analyze().
@@ -982,6 +1113,14 @@ export function removeExecutor(event
     reason: event['Removed Reason'] ?? '',
   };
   state.executors.removed.push(ev);
+  // Blocks lost with their executor get no BlockUpdated; drop them as Spark's AppStatusListener
+  // does, so a partition re-cached elsewhere isn't counted twice.
+  const suffix = `@${ev.executorId}`;
+  for (const rdd of state.rddBlocks.values()) {
+    for (const key of rdd.blocks.keys()) {
+      if (key.endsWith(suffix)) dropRddBlock(rdd, key);
+    }
+  }
   return { type: 'executor', data: ev };
 }
 
@@ -1058,6 +1197,9 @@ export function processEvent(event            , state             )          {
     case 'SparkListenerExecutorRemoved':
       return removeExecutor(event, state);
 
+    case 'SparkListenerBlockUpdated':
+      return recordBlockUpdate(event, state);
+
     default:
       return assertNever(event);
   }
@@ -1215,7 +1357,14 @@ export function dispatchLine(
   parseAndDispatch(line, state, emit);
 }
 
+// With logBlockUpdates on, most BlockUpdated lines are broadcast/shuffle blocks recordBlockUpdate
+// ignores. Spark writes "Event" first and "Block ID" as a plain string, so those lines can be
+// dropped on a substring test before paying for JSON.parse.
+const BLOCK_UPDATED_PREFIX = '{"Event":"SparkListenerBlockUpdated",';
+const RDD_BLOCK_ID_FRAGMENT = '"Block ID":"rdd_';
+
 function parseAndDispatch(line        , state             , emit                        )       {
+  if (line.startsWith(BLOCK_UPDATED_PREFIX) && !line.includes(RDD_BLOCK_ID_FRAGMENT)) return;
   let parsed         ;
   try {
     parsed = parseTaskEnd(line) ?? JSON.parse(stripPlanDescription(line));
@@ -1265,11 +1414,26 @@ export function collectStageExecutorMetrics(state             )
   return out;
 }
 
+// Speculation totals of every stage a late TaskEnd added waste to (accountLateSpeculativeLoser),
+// re-posted once before `done`: the stage message posted at completion carried the earlier totals.
+export function collectLateSpeculationWaste(
+  state             ,
+)                                                                                 {
+  const out = new Map                                                                           ();
+  for (const [id, stage] of state.stages) {
+    if (stage.lateSpeculationWaste) {
+      out.set(id, { speculationWasteMs: stage.speculationWasteMs, speculationWastedAttempts: stage.speculationWastedAttempts });
+    }
+  }
+  return out;
+}
+
 export function emitParseCompletion(state             , emit                        , linesProcessed        )       {
   // Executions that never ended keep their latest AQE update, as they did before it was deferred.
   for (const executionId of [...state.pendingAdaptiveUpdates.keys()]) flushAdaptiveUpdate(executionId, state, emit);
   emit({ type: 'progress', pct: 1, linesProcessed });
   emit({ type: 'runAggregates', data: computeRunAggregates(state.taskStore) });
+  emit({ type: 'stageSpeculationWaste', data: collectLateSpeculationWaste(state) });
   emit({ type: 'stageExecutorMetrics', data: collectStageExecutorMetrics(state) });
   emit(appMessage(state));
   emit({ type: 'done', skippedLines: state.skippedLines });

package/vendor-core/event-schemas.js

@@ -209,6 +209,26 @@ export const StageSubmittedEventSchema = z.object({
   }),
 });
 
+// recordBlockUpdate. Written only with spark.eventLog.logBlockUpdates.enabled=true; one event per
+// block status change reported to the driver's BlockManagerMaster (a removal or eviction carries
+// storage level NONE with zero sizes). Only 'rdd_<rddId>_<partition>' blocks are read.
+export const BlockUpdatedEventSchema = z.object({
+  Event: z.literal('SparkListenerBlockUpdated'),
+  'Block Updated Info': z.object({
+    'Block Manager ID': z.object({
+      'Executor ID': z.string().optional(),
+    }).optional(),
+    'Block ID': z.string(),
+    'Storage Level': z.object({
+      'Use Disk': z.boolean().optional(),
+      'Use Memory': z.boolean().optional(),
+      Replication: z.number().optional(),
+    }).optional(),
+    'Memory Size': z.number().optional(),
+    'Disk Size': z.number().optional(),
+  }),
+});
+
 // inline StageCompleted case.
 export const StageCompletedEventSchema = z.object({
   Event: z.literal('SparkListenerStageCompleted'),
@@ -345,5 +365,6 @@ export const SparkEventSchema = z.discriminatedUnion('Event', [
   DriverAccumUpdatesEventSchema,
   ExecutorAddedEventSchema,
   ExecutorRemovedEventSchema,
+  BlockUpdatedEventSchema,
 ]);