sparkforensics-server 0.2.3 → 0.3.0

package/vendor-core/event-handlers.js

@@ -19,6 +19,7 @@ import {
 } from './event-schemas.js';
 import { assertNever } from './assert-never.js';
 import { finalizeStage } from './stage-quantiles.js';
+import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
 import { computeRunAggregates } from './run-aggregates.js';
              
                                                                                         
@@ -94,6 +95,9 @@ const MAX_TASK_SAMPLES = 20;
                      
                      
                         
+                                                                                        
+                                                                      
+                                    
                        
                
                      
@@ -136,6 +140,8 @@ const MAX_TASK_SAMPLES = 20;
                                       
                                     
                                                         
+                                                                                           
+                                                        
                                        
                        
                          
@@ -485,6 +491,19 @@ function taskRecordToSample(t            )                   {
   };
 }
 
+// One shared detail object per distinct failure, so a stage with thousands of failed attempts
+// holds each bounded stack excerpt once. Past the cap an attempt keeps only its reason tag.
+function internTaskFailure(stage             , endReason                                     )                           {
+  const detail = extractTaskFailureDetail(endReason);
+  if (!detail || !stage.failureDetails) return null;
+  const key = taskFailureKey(detail);
+  const known = stage.failureDetails.get(key);
+  if (known) return known;
+  if (stage.failureDetails.size >= MAX_FAILURE_DETAILS_PER_STAGE) return null;
+  stage.failureDetails.set(key, detail);
+  return detail;
+}
+
 export function accumulateTask(event                                    , state             )       {
   const stageId = event['Stage ID'];
   const stage = state.stages.get(stageId);
@@ -521,6 +540,7 @@ export function accumulateTask(event                                    , state
     launchTime: info['Launch Time'] ?? 0,
     finishTime: info['Finish Time'] ?? 0,
     reason: event['Task End Reason']?.['Reason'] ?? null,
+    failure: failed ? internTaskFailure(stage, event['Task End Reason']) : null,
     speculative: info['Speculative'] === true,
     host: info['Host'] ?? '',
     executorId: info['Executor ID'] ?? '',
@@ -802,6 +822,7 @@ export function submitStage(event                                           , st
     failureReasons: new Map(),
     stageFailureReason: null,
     taskAttempts: new Map(),
+    failureDetails: new Map(),
     retryTaskSamples: [],
     retryWasteMs: 0,
     wastedAttempts: 0,

package/vendor-core/event-schemas.js

@@ -238,6 +238,14 @@ export const TaskEndEventSchema = z.object({
   'Stage Attempt ID': z.number().optional(),
   'Task End Reason': z.object({
     Reason: z.string().optional(),
+    // The error behind a failed attempt, read by extractTaskFailureDetail (task-failure.ts), which
+    // keeps only string values: unknown here so an unexpected shape drops the detail, never the task.
+    'Class Name': z.unknown().optional(),
+    Description: z.unknown().optional(),
+    'Full Stack Trace': z.unknown().optional(),
+    'Loss Reason': z.unknown().optional(),
+    Message: z.unknown().optional(),
+    'Kill Reason': z.unknown().optional(),
   }).optional(),
   'Task Info': z.object({
     'Launch Time': z.number().optional(),

package/vendor-core/evidence-report.js

@@ -6,6 +6,7 @@ import { detectorCatalog } from './detectors.js';
 import { typeTag, formatBytes, formatDuration, IMPACT_BAND_ORDER } from './format-utils.js';
 import { FINDING_NAMES, titleCase } from './finding-names.js';
 import { redactReport } from './redact.js';
+import { formatTaskFailureHeadline,                       } from './task-failure.js';
 import { coreFindingActionLabel } from './finding-action-label.js';
 import { matchesFindingFilterCriteria } from './finding-filter-predicate.js';
 import { buildRecommendationRollup, isEligible, rankFindings,                  } from './recommendation-rollup.js';
@@ -269,6 +270,21 @@ function renderEvidenceValue(key        , value         )         {
   return String(value);
 }
 
+// The `failures` finding's distinct errors: a headline per group, its stack excerpt as an indented
+// code block (indented, not fenced, so no excerpt content can close it early).
+function renderFailureGroups(groups                    )           {
+  const lines = [`  - failureGroups: ${groups.length}`];
+  for (const g of groups) {
+    lines.push(`    - ${g.count} task(s): ${formatTaskFailureHeadline(g)}`);
+    if (g.stackExcerpt) {
+      lines.push('');
+      for (const l of g.stackExcerpt.split('\n')) lines.push(`          ${l}`);
+      lines.push('');
+    }
+  }
+  return lines;
+}
+
 function formatWallClockRange(low        , high        )         {
   const fmtMs = (ms        ) => (ms === 0 ? '0s' : formatDuration(ms));
   return low === high ? `Estimated ${fmtMs(high)}` : `Estimated ${fmtMs(low)}-${fmtMs(high)}`;
@@ -336,7 +352,10 @@ function renderMarkdown(json                    )         {
     const evidence = Object.entries(r.evidence ?? {}).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
     if (evidence.length) {
       lines.push('- evidence:');
-      for (const [k, v] of evidence) lines.push(`  - ${k}: ${renderEvidenceValue(k, v)}`);
+      for (const [k, v] of evidence) {
+        if (k === 'failureGroups' && Array.isArray(v)) lines.push(...renderFailureGroups(v                      ));
+        else lines.push(`  - ${k}: ${renderEvidenceValue(k, v)}`);
+      }
     }
     lines.push('');
   }

package/vendor-core/impact-estimator.js

@@ -1,4 +1,5 @@
                                                                                                        
+import { nsToMs } from './format-utils.js';
 import {
   computeOccupancy, estimateSingleStage, estimateMultiStage, tailRecoveryMs, tailRemovedWorkMs, stragglerFixLongestTaskMs,
                                                                                 
@@ -82,6 +83,24 @@ const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
 // Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
 const RE_READ_THROUGHPUT_BPS = 125_000_000;
 
+// Below this share of executorRunTime spent on CPU, a stage's tasks were idle, waiting on something
+// outside Spark: on 14 real logs (2026-09-23) every non-Python stage under 1% was a JDBC read, a
+// file listing or a Delta log read, while file writes, which more partitions do parallelize,
+// start at 2%.
+const IDLE_CPU_SHARE_MAX = 0.01;
+
+// True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
+// the share can't be trusted: no CPU time recorded (older Spark logs omit the metric), or Python
+// code run through PythonRDD, whose worker-process CPU executorCpuTime (the JVM task thread's)
+// never counts (such stages read 0.1% on the same logs while computing).
+function tasksMostlyIdle(stage       )          {
+  const runMs = stage.executorRunTime ?? 0;
+  const cpuMs = nsToMs(stage.executorCpuTime ?? 0);
+  if (runMs <= 0 || cpuMs <= 0) return false;
+  if (/PythonRDD/.test(stage.name ?? '') || /org\.apache\.spark\.api\.python\./.test(stage.details ?? '')) return false;
+  return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
+}
+
 // skew and straggler claim time off the stage's longest task itself, so the occupancy clip must
 // not floor them at that same task (see estimateSingleStage). detectors.ts's clippedWasteMs gates
 // both detectors on the same option so the firing floor and the displayed estimate agree.
@@ -332,14 +351,15 @@ function computeEstimateForFinding(
       // is queueing no partition count recovers. Splitting partitions splits the longest task
       // too, hence TAIL_CLAIM's post-fix floor. Unknown cluster size: no defensible figure.
       if (totalCores <= 0) return costOnly('modeled');
-      // A stage that read no input and no shuffle has no data for more partitions to split (a
-      // 1-task count stage open 27 minutes on 5s of CPU was claimed 99% recoverable): claim 0.
+      // A stage that read no input and no shuffle, its tasks idle waiting on an external system,
+      // gains nothing from more partitions (a 1-task JDBC count stage open 27 minutes on 5s of CPU
+      // was claimed 99% recoverable): claim 0. Stages that read bytes keep their claim.
       const readBytes = (stage.inputBytes ?? 0) + (stage.shuffleReadBytes ?? 0);
       const activeMs = typeof stage.taskActiveMs === 'number'
         ? stage.taskActiveMs
         : Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
       const taskCount = stage.taskCount ?? 0;
-      const wasteMs = readBytes > 0 ? activeMs * Math.max(0, 1 - taskCount / totalCores) : 0;
+      const wasteMs = readBytes <= 0 && tasksMostlyIdle(stage) ? 0 : activeMs * Math.max(0, 1 - taskCount / totalCores);
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' }, TAIL_CLAIM);
     }
     case 'partitionSizing': {

package/vendor-core/occupancy.js

@@ -152,18 +152,19 @@ const SERIAL_GATE_THRESHOLD = 0.999;
                                
                           
                                  
+                                
                            
                            
  
 
-// Wall-clock a skew/straggler fix recovers, given the slowest task's excess over the median. A
-// lone straggler costs that excess; a tail of many slow tasks (a bimodal stage: 26% of 1400
-// tasks over 4x P50 on a real log) costs its summed excess (stragglerExcessMs) spread over the
-// slots the stage had (peakConcurrentTasks), far more than one task's. The task-level replay in
-// dev/eval-tail-replay.mjs recovers about the larger of the two. Average concurrency would be the
-// wrong divisor: a tail-dominated stage runs few tasks for most of its span (5.7 average vs 14
-// peak on one real stage), which doubled the claim.
+// Wall-clock a skew/straggler fix recovers: finalizeStage's task-level replay
+// (tailReplayRecoveryMs, computeTailReplayRecoveryMs) when the stage carries it. Without it (a
+// stage built by hand, as in detector tests) an estimate from the slowest task's excess over the
+// median: a lone straggler costs that excess; a tail of many slow tasks costs its summed excess
+// (stragglerExcessMs) spread over the slots the stage had (peakConcurrentTasks), and the replay
+// recovers about the larger of the two.
 export function tailRecoveryMs(stage           , singleTaskExcessMs        )         {
+  if (stage.tailReplayRecoveryMs != null) return stage.tailReplayRecoveryMs;
   const excessMs = stage.stragglerExcessMs ?? 0;
   const slots = stage.peakConcurrentTasks ?? 0;
   if (excessMs <= 0 || slots <= 0) return singleTaskExcessMs;

package/vendor-core/parser-worker.js

@@ -4,7 +4,9 @@ import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
 import { createSnappyBlockDecoder } from './snappy-block.js';
 import { createState, dispatchLine, buildChunkDecoder, emitParseCompletion,                                   } from './event-handlers.js';
 import { TASK_FIELD_NAMES } from './stage-quantiles.js';
-import { runParseFromUrl, sniffCodec } from './shs-fetch.js';
+import { runParseFromUrl, sniffCodec, parseZipArchive } from './shs-fetch.js';
+import { isZip } from './zip-archive.js';
+import { createWorkerZstdDecoders } from './zstd-worker-client.js';
 
 export {
   buildChunkDecoder, createState, normalizeSparkProperties, parseSparkMemoryMB, extractResources,
@@ -34,6 +36,8 @@ const MIN_PROGRESS_STEPS = 100;
 const PROGRESS_EMIT_LINES = 300;
 
                                      
+const NOT_AN_EVENT_LOG = 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.';
+
 // zstdDecoder replaces the vendored fzstd for zstd input; the Node CLI/MCP path passes
 // cli/native-zstd.ts's native-zlib decoder, which a browser bundle can't import.
                                                                                        
@@ -61,9 +65,11 @@ const PROGRESS_EMIT_LINES = 300;
 // without touching the vendored files (mirrors shs-fetch.ts's shim).
                                                                            
                                                                                            
-// A Node decoder may decompress off the main thread: streamFile awaits each push.
+// A Node decoder, or the browser's decompress worker (zstd-worker-client.ts), may decompress off
+// the calling thread: streamFile awaits each push, and calls cancel() when it abandons the stream.
                                                                             
                                                                  
+                  
   
 
 // Stream one File's (possibly compressed) bytes through the codec dispatch,
@@ -89,7 +95,7 @@ export async function streamFile(
   const gunzip = codec === 'gz' ? new (Gunzip                                   )((inflated) => onChunk(inflated, currentPct)) : null;
   const lz4 = codec === 'lz4' ? createLz4BlockDecoder((inflated) => onChunk(inflated, currentPct)) : null;
   const onZstdChunk = (inflated            ) => onChunk(inflated, currentPct);
-  const zstd = codec !== 'zstd' ? null
+  const zstd                                        = codec !== 'zstd' ? null
     : zstdDecoder ? zstdDecoder(onZstdChunk)
       : new (ZstdDecompress                                   )(onZstdChunk);
   const snappy = codec === 'snappy' ? createSnappyBlockDecoder((inflated) => onChunk(inflated, currentPct)) : null;
@@ -101,17 +107,22 @@ export async function streamFile(
   const stepSize = Math.max(1, Math.min(chunkSize, Math.ceil(file.size / MIN_PROGRESS_STEPS)));
 
   let offset = 0;
-  while (offset < file.size) {
-    const start = offset;
-    const slice = new Uint8Array(await file.slice(start, start + stepSize).arrayBuffer());
-    offset += stepSize;
-    const final = offset >= file.size;
-    currentPct = start / file.size;
-    if (gunzip) gunzip.push(slice, final);
-    else if (lz4) lz4.push(slice);
-    else if (zstd) await zstd.push(slice, final);
-    else if (snappy) snappy.push(slice);
-    else onChunk(slice, currentPct);
+  try {
+    while (offset < file.size) {
+      const start = offset;
+      const slice = new Uint8Array(await file.slice(start, start + stepSize).arrayBuffer());
+      offset += stepSize;
+      const final = offset >= file.size;
+      currentPct = start / file.size;
+      if (gunzip) gunzip.push(slice, final);
+      else if (lz4) lz4.push(slice);
+      else if (zstd) await zstd.push(slice, final);
+      else if (snappy) snappy.push(slice);
+      else onChunk(slice, currentPct);
+    }
+  } catch (e) {
+    zstd?.cancel?.();
+    throw e;
   }
   if (lz4) lz4.end();
   if (snappy) snappy.end();
@@ -132,6 +143,20 @@ export async function runParse(
     return;
   }
 
+  // A Spark History Server download (the UI's download link, or GET
+  // /api/v1/applications/<id>/logs) is a zip holding the log file, or a
+  // rolling log's parts: unwrap it through the same path the SHS fetch uses.
+  if (isZip(new Uint8Array(await file.slice(0, Math.min(4, file.size)).arrayBuffer()))) {
+    await parseZipArchive(file, state, emit, {
+      zstdDecoder,
+      chunkSize,
+      progressEvery: PROGRESS_EMIT_LINES,
+      reportPct: true,
+      onInvalid: (detail) => emit({ type: 'error', message: detail ?? NOT_AN_EVENT_LOG }),
+    });
+    return;
+  }
+
   const decoder = buildChunkDecoder();
   const joined               = [];
   let linesProcessed = 0;
@@ -160,7 +185,7 @@ export async function runParse(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
+    emit({ type: 'error', message: NOT_AN_EVENT_LOG });
     return;
   }
 
@@ -219,7 +244,7 @@ export async function runParseFiles(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
+    emit({ type: 'error', message: NOT_AN_EVENT_LOG });
     return;
   }
 
@@ -232,17 +257,25 @@ const isWorker = typeof WorkerGlobalScope !== 'undefined' && self instanceof Wor
 
 if (isWorker) {
   let workerState                     = null;
+  // Dropped zstd files decompress in a second worker, overlapping with parsing here. The
+  // `new Worker(new URL(...))` stays inline for Vite's worker detection (see ingest.ts). The SHS
+  // path (runParseFromUrl) keeps in-thread fzstd.
+  const zstdDecoder = createWorkerZstdDecoders(
+    () => new Worker(new URL('./zstd-worker.js', import.meta.url), { type: 'module' }),
+    (onChunk) => new (ZstdDecompress                                   )(onChunk),
+    { onFallback: (reason) => console.warn(`zstd: decompressing on the parse worker: ${reason}`) },
+  );
 
   self.onmessage = async ({ data }                                     ) => {
     if (data.type === 'parse') {
       workerState = createState();
-      await runParse(data.file, workerState);
+      await runParse(data.file, workerState, { zstdDecoder });
     } else if (data.type === 'parseFromUrl') {
       workerState = createState();
       await runParseFromUrl(data.request, workerState);
     } else if (data.type === 'parseFiles') {
       workerState = createState();
-      await runParseFiles(data.files, workerState);
+      await runParseFiles(data.files, workerState, { zstdDecoder });
     } else if (data.type === 'getTaskData') {
       const { stageId, reqId } = data;
       const stored = workerState?.taskStore.get(stageId) ?? new Float64Array(0);

package/vendor-core/plan-summary.js

@@ -45,7 +45,7 @@ function pushLongFilterWarning(result             , len        )       {
 }
 
 // Stable relation-identity key for a scan node: "<format>:<relation>" (e.g.
-// "delta:mx.t_emp_whitelist", "parquet:business_prd.sales", "jdbc:dw.d_producto")
+// "delta:mx.store_map", "parquet:warehouse.sales", "jdbc:dw.dim_product")
 // or null when the node is internal Delta metadata / a non-scan / un-nameable.
 // The single source of truth for scan identity, shared by `visitScan` (summary)
 // and the `cachingOpportunity` detector so the regexes live in one place.

package/vendor-core/redact.js

@@ -8,6 +8,7 @@
 // and non-mutating (returns a fresh, deep-copied tree).
 
                                                       
+import { redactTaskFailureGroup,                        } from './task-failure.js';
 
 // Host / IP identifier patterns. Used to enumerate host names that surface only
 // inside free text: recommendation strings, `stageFailed`'s failure-reason
@@ -79,6 +80,25 @@ function collectHostFields(node         , hosts             )       {
   }
 }
 
+// A failed-task error's message and stack text can carry file paths and data values that no
+// host/app-id pattern recognizes, so they are dropped rather than pseudonymized: every array under
+// a key named `failureGroups` (a `failures` finding's, or a stage's in the HTML export) gets
+// redactTaskFailureGroup applied. Walking by key name, like collectHostFields, needs no path list.
+// Returns a fresh tree.
+function redactFailureGroups   (node   )    {
+  if (Array.isArray(node)) return node.map((n) => redactFailureGroups(n))     ;
+  if (node && typeof node === 'object') {
+    const out                          = {};
+    for (const [k, v] of Object.entries(node)) {
+      out[k] = k === 'failureGroups' && Array.isArray(v)
+        ? v.map((g) => (g && typeof g === 'object' ? redactTaskFailureGroup(g                     ) : g))
+        : redactFailureGroups(v);
+    }
+    return out     ;
+  }
+  return node;
+}
+
 // Spark config keys ending in `host`/`hostname` (e.g. spark.driver.host,
 // spark.yarn.am.hostname) carry plain FQDN host names that neither
 // HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
@@ -172,7 +192,8 @@ function applyReplacements   (node   , ids
   return deepReplace(node, merged)     ;
 }
 
-export function redactReport                            (report   )    {
+export function redactReport                            (input   )    {
+  const report = redactFailureGroups(input);
   const { appIds, hosts } = collectIds(report);
   return applyReplacements(report, { appIds, hosts });
 }
@@ -216,7 +237,8 @@ export function redactComparison   (comparison   )    {
 // this also walks executors.added/removed for their literal `host` field
 // (ExecutorAddedEvent.host), since raw executor records: not just findings
 //: reach data.js.
-export function redactExportData(data               )                {
+export function redactExportData(input               )                {
+  const data = redactFailureGroups(input);
   const appIds = new Set        ();
   const hosts = new Set        ();
   const appId = data.app?.id;

package/vendor-core/run-comparison.js

@@ -180,6 +180,14 @@ function skewRatios(stages         )           {
   return out;
 }
 
+// Every key metricDeltas() emits, in emission order. The CLI validates
+// --regression-metric against it, so a misspelled key is a usage error rather
+// than an inconclusive budget. A contract test keeps it in sync.
+export const COMPARISON_METRIC_KEYS                    = [
+  'wallClock', 'shuffleSpill', 'taskSkew', 'failedTaskRate', 'diskSpill', 'gcTime',
+  'inputBytes', 'outputBytes', 'executorRunTime', 'taskCount', 'executorsAdded',
+];
+
 // Volume/count metrics, not cost metrics: more or less input/output data, or
 // tasks/executors, isn't inherently better or worse (it may just reflect a
 // differently-sized job), unlike wall-clock, spill, GC, etc. Exported as the