sparkforensics-server 0.2.4 → 0.3.0

package/vendor-core/event-handlers.js

@@ -19,6 +19,7 @@ import {
 } from './event-schemas.js';
 import { assertNever } from './assert-never.js';
 import { finalizeStage } from './stage-quantiles.js';
+import { MAX_FAILURE_DETAILS_PER_STAGE, extractTaskFailureDetail, taskFailureKey,                        } from './task-failure.js';
 import { computeRunAggregates } from './run-aggregates.js';
              
                                                                                         
@@ -94,6 +95,9 @@ const MAX_TASK_SAMPLES = 20;
                      
                      
                         
+                                                                                        
+                                                                      
+                                    
                        
                
                      
@@ -136,6 +140,8 @@ const MAX_TASK_SAMPLES = 20;
                                       
                                     
                                                         
+                                                                                           
+                                                        
                                        
                        
                          
@@ -485,6 +491,19 @@ function taskRecordToSample(t            )                   {
   };
 }
 
+// One shared detail object per distinct failure, so a stage with thousands of failed attempts
+// holds each bounded stack excerpt once. Past the cap an attempt keeps only its reason tag.
+function internTaskFailure(stage             , endReason                                     )                           {
+  const detail = extractTaskFailureDetail(endReason);
+  if (!detail || !stage.failureDetails) return null;
+  const key = taskFailureKey(detail);
+  const known = stage.failureDetails.get(key);
+  if (known) return known;
+  if (stage.failureDetails.size >= MAX_FAILURE_DETAILS_PER_STAGE) return null;
+  stage.failureDetails.set(key, detail);
+  return detail;
+}
+
 export function accumulateTask(event                                    , state             )       {
   const stageId = event['Stage ID'];
   const stage = state.stages.get(stageId);
@@ -521,6 +540,7 @@ export function accumulateTask(event                                    , state
     launchTime: info['Launch Time'] ?? 0,
     finishTime: info['Finish Time'] ?? 0,
     reason: event['Task End Reason']?.['Reason'] ?? null,
+    failure: failed ? internTaskFailure(stage, event['Task End Reason']) : null,
     speculative: info['Speculative'] === true,
     host: info['Host'] ?? '',
     executorId: info['Executor ID'] ?? '',
@@ -802,6 +822,7 @@ export function submitStage(event                                           , st
     failureReasons: new Map(),
     stageFailureReason: null,
     taskAttempts: new Map(),
+    failureDetails: new Map(),
     retryTaskSamples: [],
     retryWasteMs: 0,
     wastedAttempts: 0,

package/vendor-core/event-schemas.js

@@ -238,6 +238,14 @@ export const TaskEndEventSchema = z.object({
   'Stage Attempt ID': z.number().optional(),
   'Task End Reason': z.object({
     Reason: z.string().optional(),
+    // The error behind a failed attempt, read by extractTaskFailureDetail (task-failure.ts), which
+    // keeps only string values: unknown here so an unexpected shape drops the detail, never the task.
+    'Class Name': z.unknown().optional(),
+    Description: z.unknown().optional(),
+    'Full Stack Trace': z.unknown().optional(),
+    'Loss Reason': z.unknown().optional(),
+    Message: z.unknown().optional(),
+    'Kill Reason': z.unknown().optional(),
   }).optional(),
   'Task Info': z.object({
     'Launch Time': z.number().optional(),

package/vendor-core/evidence-report.js

@@ -6,6 +6,7 @@ import { detectorCatalog } from './detectors.js';
 import { typeTag, formatBytes, formatDuration, IMPACT_BAND_ORDER } from './format-utils.js';
 import { FINDING_NAMES, titleCase } from './finding-names.js';
 import { redactReport } from './redact.js';
+import { formatTaskFailureHeadline,                       } from './task-failure.js';
 import { coreFindingActionLabel } from './finding-action-label.js';
 import { matchesFindingFilterCriteria } from './finding-filter-predicate.js';
 import { buildRecommendationRollup, isEligible, rankFindings,                  } from './recommendation-rollup.js';
@@ -269,6 +270,21 @@ function renderEvidenceValue(key        , value         )         {
   return String(value);
 }
 
+// The `failures` finding's distinct errors: a headline per group, its stack excerpt as an indented
+// code block (indented, not fenced, so no excerpt content can close it early).
+function renderFailureGroups(groups                    )           {
+  const lines = [`  - failureGroups: ${groups.length}`];
+  for (const g of groups) {
+    lines.push(`    - ${g.count} task(s): ${formatTaskFailureHeadline(g)}`);
+    if (g.stackExcerpt) {
+      lines.push('');
+      for (const l of g.stackExcerpt.split('\n')) lines.push(`          ${l}`);
+      lines.push('');
+    }
+  }
+  return lines;
+}
+
 function formatWallClockRange(low        , high        )         {
   const fmtMs = (ms        ) => (ms === 0 ? '0s' : formatDuration(ms));
   return low === high ? `Estimated ${fmtMs(high)}` : `Estimated ${fmtMs(low)}-${fmtMs(high)}`;
@@ -336,7 +352,10 @@ function renderMarkdown(json                    )         {
     const evidence = Object.entries(r.evidence ?? {}).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
     if (evidence.length) {
       lines.push('- evidence:');
-      for (const [k, v] of evidence) lines.push(`  - ${k}: ${renderEvidenceValue(k, v)}`);
+      for (const [k, v] of evidence) {
+        if (k === 'failureGroups' && Array.isArray(v)) lines.push(...renderFailureGroups(v                      ));
+        else lines.push(`  - ${k}: ${renderEvidenceValue(k, v)}`);
+      }
     }
     lines.push('');
   }

package/vendor-core/parser-worker.js

@@ -4,7 +4,8 @@ import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
 import { createSnappyBlockDecoder } from './snappy-block.js';
 import { createState, dispatchLine, buildChunkDecoder, emitParseCompletion,                                   } from './event-handlers.js';
 import { TASK_FIELD_NAMES } from './stage-quantiles.js';
-import { runParseFromUrl, sniffCodec } from './shs-fetch.js';
+import { runParseFromUrl, sniffCodec, parseZipArchive } from './shs-fetch.js';
+import { isZip } from './zip-archive.js';
 import { createWorkerZstdDecoders } from './zstd-worker-client.js';
 
 export {
@@ -35,6 +36,8 @@ const MIN_PROGRESS_STEPS = 100;
 const PROGRESS_EMIT_LINES = 300;
 
                                      
+const NOT_AN_EVENT_LOG = 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.';
+
 // zstdDecoder replaces the vendored fzstd for zstd input; the Node CLI/MCP path passes
 // cli/native-zstd.ts's native-zlib decoder, which a browser bundle can't import.
                                                                                        
@@ -140,6 +143,20 @@ export async function runParse(
     return;
   }
 
+  // A Spark History Server download (the UI's download link, or GET
+  // /api/v1/applications/<id>/logs) is a zip holding the log file, or a
+  // rolling log's parts: unwrap it through the same path the SHS fetch uses.
+  if (isZip(new Uint8Array(await file.slice(0, Math.min(4, file.size)).arrayBuffer()))) {
+    await parseZipArchive(file, state, emit, {
+      zstdDecoder,
+      chunkSize,
+      progressEvery: PROGRESS_EMIT_LINES,
+      reportPct: true,
+      onInvalid: (detail) => emit({ type: 'error', message: detail ?? NOT_AN_EVENT_LOG }),
+    });
+    return;
+  }
+
   const decoder = buildChunkDecoder();
   const joined               = [];
   let linesProcessed = 0;
@@ -168,7 +185,7 @@ export async function runParse(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
+    emit({ type: 'error', message: NOT_AN_EVENT_LOG });
     return;
   }
 
@@ -227,7 +244,7 @@ export async function runParseFiles(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
+    emit({ type: 'error', message: NOT_AN_EVENT_LOG });
     return;
   }
 
@@ -242,7 +259,7 @@ if (isWorker) {
   let workerState                     = null;
   // Dropped zstd files decompress in a second worker, overlapping with parsing here. The
   // `new Worker(new URL(...))` stays inline for Vite's worker detection (see ingest.ts). The SHS
-  // path (runParseFromUrl) decodes whole zip entries synchronously and keeps in-thread fzstd.
+  // path (runParseFromUrl) keeps in-thread fzstd.
   const zstdDecoder = createWorkerZstdDecoders(
     () => new Worker(new URL('./zstd-worker.js', import.meta.url), { type: 'module' }),
     (onChunk) => new (ZstdDecompress                                   )(onChunk),

package/vendor-core/redact.js

@@ -8,6 +8,7 @@
 // and non-mutating (returns a fresh, deep-copied tree).
 
                                                       
+import { redactTaskFailureGroup,                        } from './task-failure.js';
 
 // Host / IP identifier patterns. Used to enumerate host names that surface only
 // inside free text: recommendation strings, `stageFailed`'s failure-reason
@@ -79,6 +80,25 @@ function collectHostFields(node         , hosts             )       {
   }
 }
 
+// A failed-task error's message and stack text can carry file paths and data values that no
+// host/app-id pattern recognizes, so they are dropped rather than pseudonymized: every array under
+// a key named `failureGroups` (a `failures` finding's, or a stage's in the HTML export) gets
+// redactTaskFailureGroup applied. Walking by key name, like collectHostFields, needs no path list.
+// Returns a fresh tree.
+function redactFailureGroups   (node   )    {
+  if (Array.isArray(node)) return node.map((n) => redactFailureGroups(n))     ;
+  if (node && typeof node === 'object') {
+    const out                          = {};
+    for (const [k, v] of Object.entries(node)) {
+      out[k] = k === 'failureGroups' && Array.isArray(v)
+        ? v.map((g) => (g && typeof g === 'object' ? redactTaskFailureGroup(g                     ) : g))
+        : redactFailureGroups(v);
+    }
+    return out     ;
+  }
+  return node;
+}
+
 // Spark config keys ending in `host`/`hostname` (e.g. spark.driver.host,
 // spark.yarn.am.hostname) carry plain FQDN host names that neither
 // HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
@@ -172,7 +192,8 @@ function applyReplacements   (node   , ids
   return deepReplace(node, merged)     ;
 }
 
-export function redactReport                            (report   )    {
+export function redactReport                            (input   )    {
+  const report = redactFailureGroups(input);
   const { appIds, hosts } = collectIds(report);
   return applyReplacements(report, { appIds, hosts });
 }
@@ -216,7 +237,8 @@ export function redactComparison   (comparison   )    {
 // this also walks executors.added/removed for their literal `host` field
 // (ExecutorAddedEvent.host), since raw executor records: not just findings
 //: reach data.js.
-export function redactExportData(data               )                {
+export function redactExportData(input               )                {
+  const data = redactFailureGroups(input);
   const appIds = new Set        ();
   const hosts = new Set        ();
   const appId = data.app?.id;

package/vendor-core/run-comparison.js

@@ -180,6 +180,14 @@ function skewRatios(stages         )           {
   return out;
 }
 
+// Every key metricDeltas() emits, in emission order. The CLI validates
+// --regression-metric against it, so a misspelled key is a usage error rather
+// than an inconclusive budget. A contract test keeps it in sync.
+export const COMPARISON_METRIC_KEYS                    = [
+  'wallClock', 'shuffleSpill', 'taskSkew', 'failedTaskRate', 'diskSpill', 'gcTime',
+  'inputBytes', 'outputBytes', 'executorRunTime', 'taskCount', 'executorsAdded',
+];
+
 // Volume/count metrics, not cost metrics: more or less input/output data, or
 // tasks/executors, isn't inherently better or worse (it may just reflect a
 // differently-sized job), unlike wall-clock, spill, GC, etc. Exported as the

package/vendor-core/shs-fetch.js

@@ -1,4 +1,4 @@
-import { unzipSync, Gunzip } from './vendor/fflate.js';
+import { Gunzip } from './vendor/fflate.js';
 import { createLz4BlockDecoder } from './lz4-block.js';
 import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
 import { createSnappyBlockDecoder } from './snappy-block.js';
@@ -6,6 +6,8 @@ import { buildProxyRequestUrl, isShsErrorCode } from './shs-request.js';
 import { dispatchLine, buildChunkDecoder, emitParseCompletion,                                   } from './event-handlers.js';
 import { ShsProxyErrorBodySchema } from './shs-schemas.js';
 import { naturalCompare, reassembleRollingEntries } from './rolling-log-reassembly.js';
+import { listZipEntries, streamZipEntry,                               } from './zip-archive.js';
+                                                             
 
 export { naturalCompare, reassembleRollingEntries };
 
@@ -24,48 +26,84 @@ export function sniffCodec(bytes            )
   return null;
 }
 
-// Streams one already-in-memory zip entry through its codec's block-by-block
-// decoder, invoking `onChunk` per decompressed piece; never materializes the
-// whole decompressed entry as one buffer. A giant single-segment zstd frame
-// (or gzip member) declares its full content size in the header; one-shotting
-// it (fzstd's `decompress()`, fflate's `gunzipSync()`) allocates that entire
-// size as a single ArrayBuffer up front, which can exceed what the browser
-// will allocate for a multi-GB event log ("Array buffer allocation failed").
+// Decodes one zip entry's (possibly compressed) bytes as they stream out of
+// the archive, block by block, invoking `onChunk` per decompressed piece;
+// never materializes the whole decompressed entry as one buffer. A giant
+// single-segment zstd frame (or gzip member) declares its full content size in
+// the header; one-shotting it (fzstd's `decompress()`, fflate's `gunzipSync()`)
+// allocates that entire size as a single ArrayBuffer up front, which can exceed
+// what the browser will allocate for a multi-GB event log ("Array buffer
+// allocation failed"). The codec is sniffed from the entry's first 8 bytes,
+// falling back to the entry name's suffix.
 // fflate's Gunzip and fzstd's Decompress are untyped vendor JS, so TS can't
 // infer a construct signature for them; this local shape types the call sites
 // without touching the vendored files.
                                                                            
                                                                                            
-// Like parser-worker.ts's RunOpts.zstdDecoder, but synchronous: decodeEntry never awaits push(),
-// so its result is typed `undefined` (not `void`, which would also accept an async decoder's
-// Promise). Node callers pass cli/native-zstd.ts's createNativeZstdDecoder (nodeArchiveCodecs).
-                                                                                                                            
-
-function decodeEntry(
-  name        , raw            , onChunk                             , zstdDecoder                     ,
-)       {
-  const codec = sniffCodec(raw);
+                                                                                                    
+
+function openCodecSink(
+  codec                               , name        , onChunk                             , zstdDecoder                     ,
+)            {
   if (codec === 'lz4' || name.endsWith('.lz4')) {
     const lz4 = createLz4BlockDecoder(onChunk);
-    lz4.push(raw);
-    lz4.end();
-  } else if (codec === 'gz' || name.endsWith('.gz')) {
-    new (Gunzip                                   )(onChunk).push(raw, true);
-  } else if (codec === 'zstd' || name.endsWith('.zstd') || name.endsWith('.zst')) {
-    (zstdDecoder ? zstdDecoder(onChunk) : new (ZstdDecompress                                   )(onChunk)).push(raw, true);
-  } else if (codec === 'snappy' || name.endsWith('.snappy')) {
+    return { push(chunk, final) { lz4.push(chunk); if (final) lz4.end(); } };
+  }
+  if (codec === 'gz' || name.endsWith('.gz')) {
+    const gunzip = new (Gunzip                                   )(onChunk);
+    return { push: (chunk, final) => gunzip.push(chunk, final) };
+  }
+  if (codec === 'zstd' || name.endsWith('.zstd') || name.endsWith('.zst')) {
+    const zstd = zstdDecoder ? zstdDecoder(onChunk) : new (ZstdDecompress                                   )(onChunk);
+    return { push: (chunk, final) => zstd.push(chunk, final), cancel: () => (zstd                       ).cancel?.() };
+  }
+  if (codec === 'snappy' || name.endsWith('.snappy')) {
     const snappy = createSnappyBlockDecoder(onChunk);
-    snappy.push(raw);
-    snappy.end();
-  } else {
-    onChunk(raw);
+    return { push(chunk, final) { snappy.push(chunk); if (final) snappy.end(); } };
   }
+  return { push: (chunk) => onChunk(chunk) };
+}
+
+// Wraps openCodecSink for a stream of unknown length: holds back the latest
+// chunk so the last one can be pushed with `final` set (codecs reject a
+// trailing empty push less uniformly than a real last chunk), and buffers the
+// start of the entry until it has the 8 bytes sniffCodec needs.
+function createEntryDecoder(name        , onChunk                             , zstdDecoder                     ) {
+  let head = new Uint8Array(0);
+  let held                    = null;
+  let sink                   = null;
+  const open = () => {
+    sink = openCodecSink(sniffCodec(head), name, onChunk, zstdDecoder);
+    held = head;
+  };
+  return {
+    async push(chunk            )                {
+      if (!sink) {
+        const joined = new Uint8Array(head.length + chunk.length);
+        joined.set(head);
+        joined.set(chunk, head.length);
+        head = joined;
+        if (head.length >= 8) open();
+        return;
+      }
+      await sink.push(held , false);
+      held = chunk;
+    },
+    async end()                {
+      if (!sink) {
+        if (!head.length) return;
+        open();
+      }
+      await sink .push(held , true);
+    },
+    cancel()       { sink?.cancel?.(); },
+  };
 }
 
                                      
 
-function emitShsError(emit        , code        )       {
-  emit({ type: 'error', source: 'shs', code });
+function emitShsError(emit        , code        , message         )       {
+  emit({ type: 'error', source: 'shs', code, ...(message ? { message } : {}) });
 }
 
 export async function runParseFromUrl(
@@ -141,55 +179,127 @@ export async function runParseFromUrl(
   if (total) {
     emit({ type: 'progress', pct: 0.5, linesProcessed: 0 });
   }
-  decodeShsArchive(zipBytes, state, emit);
+  await decodeShsArchive(zipBytes, state, emit);
+}
+
+// Reads the in-memory SHS download through the same random-access interface a
+// dropped File offers, so both paths share parseZipArchive.
+function bytesZipSource(bytes            )            {
+  return {
+    size: bytes.length,
+    slice: (start, end) => ({ arrayBuffer: async () => bytes.slice(start, end).buffer }),
+  };
 }
 
 export function decodeShsArchive(
   zipBytes            , state             , emit        , { zstdDecoder }                                       = {},
-)       {
-  let entries                            ;
+)                {
+  return parseZipArchive(bytesZipSource(zipBytes), state, emit, {
+    zstdDecoder,
+    onInvalid: (detail) => emitShsError(emit, 'invalid-event-log', detail ?? undefined),
+  });
+}
+
+const baseName = (e          ) => e.name.slice(e.name.lastIndexOf('/') + 1);
+const isRollingPart = (e          ) => /^events_\d+_/.test(baseName(e));
+
+// The application attempts a History Server zip holds. Without an attempt ID
+// the History Server packs every attempt into one download: each single-file
+// log is its own root entry, each rolling log its own `eventlog_v2_` directory.
+// Directory entries and `appstatus` markers are not attempts.
+function countLogAttempts(entries            )         {
+  const attempts = new Set        ();
+  for (const e of entries) {
+    if (e.name.endsWith('/') || baseName(e).toLowerCase().startsWith('appstatus')) continue;
+    const dir = e.name.slice(0, e.name.indexOf('/') + 1);
+    attempts.add(dir || (isRollingPart(e) ? '' : e.name));
+  }
+  return attempts.size;
+}
+
+// The event-log entries a single-attempt History Server zip holds, in parse
+// order. A rolling log's parts sit under an `eventlog_v2_<appId>/` directory
+// entry; they are matched and reassembled by base name. A single-file log is
+// one entry at the root. Directory entries and the rolling log's `appstatus`
+// marker are skipped.
+function selectLogEntries(entries            )             {
+  const files = entries.filter((e) => !e.name.endsWith('/'));
+  if (files.some(isRollingPart)) {
+    const byBase = new Map(files.map((e) => [baseName(e), e]));
+    return reassembleRollingEntries(files.map(baseName)).map((n) => byBase.get(n) );
+  }
+  return files.filter((e) => baseName(e).toLowerCase() !== 'appstatus');
+}
+
+                               
+                                   
+                     
+                                                                            
+                                                                             
+                         
+                                             
+                                         
+                         
+                                                                             
+                      
+ 
+
+// Parses a single-entry or rolling Spark History Server zip, streaming each
+// entry out of `source` in `chunkSize` slices. One NDJSON line decoder spans
+// all entries, since a file-roll boundary need not fall on a line boundary.
+export async function parseZipArchive(
+  source           ,
+  state             ,
+  emit        ,
+  { zstdDecoder, chunkSize = 512 * 1024, onInvalid, progressEvery = 2000, reportPct = false }              ,
+)                {
+  let logEntries            ;
+  let attempts        ;
   try {
-    entries = unzipSync(zipBytes);
-  } catch {
-    emitShsError(emit, 'invalid-event-log');
+    const entries = await listZipEntries(source);
+    attempts = countLogAttempts(entries);
+    logEntries = attempts > 1 ? [] : selectLogEntries(entries);
+  } catch (e) {
+    onInvalid(`Could not read the zip archive: ${e instanceof Error ? e.message : String(e)}`);
     return;
   }
-  const allNames = Object.keys(entries);
-  const isRolling = allNames.some(n => /^events_\d+_/.test(n));
-  let names;
-  if (isRolling) {
-    try {
-      names = reassembleRollingEntries(allNames);
-    } catch {
-      emitShsError(emit, 'invalid-event-log');
-      return;
-    }
-  } else {
-    names = allNames.filter(n => n.toLowerCase() !== 'appstatus').sort(naturalCompare);
+  if (attempts > 1) {
+    onInvalid(`The zip archive holds ${attempts} application attempts. Download a single attempt, for example GET /api/v1/applications/<appId>/<attemptId>/logs.`);
+    return;
   }
-  if (names.length === 0) {
-    emitShsError(emit, 'invalid-event-log');
+  if (logEntries.length === 0) {
+    onInvalid('The zip archive contains no event log.');
     return;
   }
 
+  const totalBytes = logEntries.reduce((sum, e) => sum + e.compressedSize, 0);
+  let bytesRead = 0;
   const decoder = buildChunkDecoder();
   const joined               = [];
   let linesProcessed = 0;
-  for (const name of names) {
+  const feed = (bytes            ) => {
+    joined.length = 0;
+    const lines = decoder.decode(bytes, joined);
+    for (let i = 0, j = 0; i < lines.length; i++) {
+      dispatchLine(lines[i], state, emit, joined[j]?.index === i ? joined[j++] : undefined);
+      linesProcessed++;
+      if (linesProcessed % progressEvery === 0) {
+        emit({ type: 'progress', pct: reportPct && totalBytes ? bytesRead / totalBytes : null, linesProcessed });
+      }
+    }
+  };
+
+  for (const entry of logEntries) {
+    const entryDecoder = createEntryDecoder(entry.name, feed, zstdDecoder);
     try {
-      decodeEntry(name, entries[name], (bytes) => {
-        joined.length = 0;
-        const lines = decoder.decode(bytes, joined);
-        for (let i = 0, j = 0; i < lines.length; i++) {
-          dispatchLine(lines[i], state, emit, joined[j]?.index === i ? joined[j++] : undefined);
-          linesProcessed++;
-          if (linesProcessed % 2000 === 0) {
-            emit({ type: 'progress', pct: null, linesProcessed });
-          }
-        }
-      }, zstdDecoder);
-    } catch {
-      emitShsError(emit, 'invalid-event-log');
+      await streamZipEntry(source, entry, (chunk) => entryDecoder.push(chunk), {
+        chunkSize,
+        onRead: (bytes) => { bytesRead += bytes; },
+      });
+      await entryDecoder.end();
+    } catch (e) {
+      entryDecoder.cancel();
+      onInvalid(`Could not decompress "${entry.name}" in the zip archive: ${e instanceof Error ? e.message : String(e)}`);
       return;
     }
   }
@@ -198,7 +308,7 @@ export function decodeShsArchive(
   }
 
   if (!state.app) {
-    emitShsError(emit, 'invalid-event-log');
+    onInvalid(null);
     return;
   }
   emitParseCompletion(state, emit, linesProcessed);

package/vendor-core/shs-load.js

@@ -19,7 +19,7 @@ export const DEFAULT_IDLE_TIMEOUT_MS = envInt('SPARKFORENSICS_SHS_TIMEOUT_MS', 3
 
 function collectShsAppModel(zipBytes            )                                                        {
   return collectViaDispatch(
-    (state, emit) => decodeShsArchive(zipBytes, state, emit, nodeArchiveCodecs),
+    (state, emit, reject) => { decodeShsArchive(zipBytes, state, emit, nodeArchiveCodecs).catch(reject); },
     (msg) => {
       const m = msg                                       ;
       return mcpError(m?.code ?? 'invalid-event-log', m?.message ?? 'Failed to decode SHS archive.');

package/vendor-core/stage-quantiles.js

@@ -1,4 +1,5 @@
                                                             
+                                                                             
 
 // Single source of truth for the packed per-task numeric array: FIELDS (offset
 // constants), TASK_FIELD_NAMES (display labels), and the finalizeStage hot-loop
@@ -74,6 +75,8 @@ export function finalizeStage(
   const executorStats = new Map();
   const failureReasons = new Map();
   const failedTaskSamples                     = [];
+  // Keyed by the interned detail object: accumulateTask shares one per distinct failure.
+  const failureGroups = new Map                           ();
   const localityStats = new Map();
   let peakExecutionMemoryMax = 0;
 
@@ -82,6 +85,8 @@ export function finalizeStage(
     if (t.failed) {
       failedTasks++;
       if (t.reason) failureReasons.set(t.reason, (failureReasons.get(t.reason) ?? 0) + 1);
+      const failure = (t                                                     ).failure;
+      if (failure) failureGroups.set(failure, (failureGroups.get(failure) ?? 0) + 1);
       if (failedTaskSamples.length < MAX_TASK_SAMPLES) {
         failedTaskSamples.push({
           taskId: t.taskId, attemptNumber: t.attemptNumber, host: t.host, executorId: t.executorId,
@@ -125,6 +130,7 @@ export function finalizeStage(
   stage.failedTasks = failedTasks;
   stage.speculativeTasks = speculativeTasks;
   stage.taskAttempts = null; // no longer needed after finalize, freeing memory
+  stage.failureDetails = null;
 
   const arr = new Float64Array(buf);
   state.taskStore.set(stageId, arr);
@@ -164,6 +170,10 @@ export function finalizeStage(
   const failureReasonsArr = [...failureReasons.entries()].map(
     ([reason, count]) => ({ reason, count })
   );
+  // Most frequent first; ties keep first-seen order (Array.prototype.sort is stable).
+  const failureGroupsArr                     = [...failureGroups.entries()]
+    .map(([detail, count]) => ({ ...detail, count }))
+    .sort((a, b) => b.count - a.count);
   const localityStatsArr = [...localityStats.entries()].map(
     ([locality, count]) => ({ locality, count })
   );
@@ -173,6 +183,7 @@ export function finalizeStage(
     hostStats: hostStatsArr, executorStats: executorStatsArr, failureReasons: failureReasonsArr, localityStats: localityStatsArr, stragglerCount, stragglerExcessMs, longestNonStragglerMs,
     tailReplayRecoveryMs,
     failedTaskSamples,
+    failureGroups: failureGroupsArr,
     peakExecutionMemoryMax,
     taskActiveMs: computeTaskActiveMs(arr),
     peakConcurrentTasks,
@@ -187,6 +198,7 @@ export function finalizeStage(
     stageType: acc.shuffleReadBytes > 0 ? 'REDUCE' : 'MAP',
   };
   delete data.taskAttempts; // internal-only field, already nulled above; never part of the public message
+  delete data.failureDetails; // internal-only intern table, summarized by failureGroups
 
   return { type: 'stage', data };
 }