sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/shs-fetch.js

@@ -1,4 +1,4 @@
-import { unzipSync, Gunzip } from './vendor/fflate.js';
+import { Gunzip } from './vendor/fflate.js';
 import { createLz4BlockDecoder } from './lz4-block.js';
 import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
 import { createSnappyBlockDecoder } from './snappy-block.js';
@@ -6,6 +6,8 @@ import { buildProxyRequestUrl, isShsErrorCode } from './shs-request.js';
 import { dispatchLine, buildChunkDecoder, emitParseCompletion,                                   } from './event-handlers.js';
 import { ShsProxyErrorBodySchema } from './shs-schemas.js';
 import { naturalCompare, reassembleRollingEntries } from './rolling-log-reassembly.js';
+import { listZipEntries, streamZipEntry,                               } from './zip-archive.js';
+                                                             
 
 export { naturalCompare, reassembleRollingEntries };
 
@@ -24,48 +26,84 @@ export function sniffCodec(bytes            )
   return null;
 }
 
-// Streams one already-in-memory zip entry through its codec's block-by-block
-// decoder, invoking `onChunk` per decompressed piece; never materializes the
-// whole decompressed entry as one buffer. A giant single-segment zstd frame
-// (or gzip member) declares its full content size in the header; one-shotting
-// it (fzstd's `decompress()`, fflate's `gunzipSync()`) allocates that entire
-// size as a single ArrayBuffer up front, which can exceed what the browser
-// will allocate for a multi-GB event log ("Array buffer allocation failed").
+// Decodes one zip entry's (possibly compressed) bytes as they stream out of
+// the archive, block by block, invoking `onChunk` per decompressed piece;
+// never materializes the whole decompressed entry as one buffer. A giant
+// single-segment zstd frame (or gzip member) declares its full content size in
+// the header; one-shotting it (fzstd's `decompress()`, fflate's `gunzipSync()`)
+// allocates that entire size as a single ArrayBuffer up front, which can exceed
+// what the browser will allocate for a multi-GB event log ("Array buffer
+// allocation failed"). The codec is sniffed from the entry's first 8 bytes,
+// falling back to the entry name's suffix.
 // fflate's Gunzip and fzstd's Decompress are untyped vendor JS, so TS can't
 // infer a construct signature for them; this local shape types the call sites
 // without touching the vendored files.
                                                                            
                                                                                            
-// Like parser-worker.ts's RunOpts.zstdDecoder, but synchronous: decodeEntry never awaits push(),
-// so its result is typed `undefined` (not `void`, which would also accept an async decoder's
-// Promise). Node callers pass cli/native-zstd.ts's createNativeZstdDecoder (nodeArchiveCodecs).
-                                                                                                                            
-
-function decodeEntry(
-  name        , raw            , onChunk                             , zstdDecoder                     ,
-)       {
-  const codec = sniffCodec(raw);
+                                                                                                    
+
+function openCodecSink(
+  codec                               , name        , onChunk                             , zstdDecoder                     ,
+)            {
   if (codec === 'lz4' || name.endsWith('.lz4')) {
     const lz4 = createLz4BlockDecoder(onChunk);
-    lz4.push(raw);
-    lz4.end();
-  } else if (codec === 'gz' || name.endsWith('.gz')) {
-    new (Gunzip                                   )(onChunk).push(raw, true);
-  } else if (codec === 'zstd' || name.endsWith('.zstd') || name.endsWith('.zst')) {
-    (zstdDecoder ? zstdDecoder(onChunk) : new (ZstdDecompress                                   )(onChunk)).push(raw, true);
-  } else if (codec === 'snappy' || name.endsWith('.snappy')) {
+    return { push(chunk, final) { lz4.push(chunk); if (final) lz4.end(); } };
+  }
+  if (codec === 'gz' || name.endsWith('.gz')) {
+    const gunzip = new (Gunzip                                   )(onChunk);
+    return { push: (chunk, final) => gunzip.push(chunk, final) };
+  }
+  if (codec === 'zstd' || name.endsWith('.zstd') || name.endsWith('.zst')) {
+    const zstd = zstdDecoder ? zstdDecoder(onChunk) : new (ZstdDecompress                                   )(onChunk);
+    return { push: (chunk, final) => zstd.push(chunk, final), cancel: () => (zstd                       ).cancel?.() };
+  }
+  if (codec === 'snappy' || name.endsWith('.snappy')) {
     const snappy = createSnappyBlockDecoder(onChunk);
-    snappy.push(raw);
-    snappy.end();
-  } else {
-    onChunk(raw);
+    return { push(chunk, final) { snappy.push(chunk); if (final) snappy.end(); } };
   }
+  return { push: (chunk) => onChunk(chunk) };
+}
+
+// Wraps openCodecSink for a stream of unknown length: holds back the latest
+// chunk so the last one can be pushed with `final` set (codecs reject a
+// trailing empty push less uniformly than a real last chunk), and buffers the
+// start of the entry until it has the 8 bytes sniffCodec needs.
+function createEntryDecoder(name        , onChunk                             , zstdDecoder                     ) {
+  let head = new Uint8Array(0);
+  let held                    = null;
+  let sink                   = null;
+  const open = () => {
+    sink = openCodecSink(sniffCodec(head), name, onChunk, zstdDecoder);
+    held = head;
+  };
+  return {
+    async push(chunk            )                {
+      if (!sink) {
+        const joined = new Uint8Array(head.length + chunk.length);
+        joined.set(head);
+        joined.set(chunk, head.length);
+        head = joined;
+        if (head.length >= 8) open();
+        return;
+      }
+      await sink.push(held , false);
+      held = chunk;
+    },
+    async end()                {
+      if (!sink) {
+        if (!head.length) return;
+        open();
+      }
+      await sink .push(held , true);
+    },
+    cancel()       { sink?.cancel?.(); },
+  };
 }
 
                                      
 
-function emitShsError(emit        , code        )       {
-  emit({ type: 'error', source: 'shs', code });
+function emitShsError(emit        , code        , message         )       {
+  emit({ type: 'error', source: 'shs', code, ...(message ? { message } : {}) });
 }
 
 export async function runParseFromUrl(
@@ -141,55 +179,127 @@ export async function runParseFromUrl(
   if (total) {
     emit({ type: 'progress', pct: 0.5, linesProcessed: 0 });
   }
-  decodeShsArchive(zipBytes, state, emit);
+  await decodeShsArchive(zipBytes, state, emit);
+}
+
+// Reads the in-memory SHS download through the same random-access interface a
+// dropped File offers, so both paths share parseZipArchive.
+function bytesZipSource(bytes            )            {
+  return {
+    size: bytes.length,
+    slice: (start, end) => ({ arrayBuffer: async () => bytes.slice(start, end).buffer }),
+  };
 }
 
 export function decodeShsArchive(
   zipBytes            , state             , emit        , { zstdDecoder }                                       = {},
-)       {
-  let entries                            ;
+)                {
+  return parseZipArchive(bytesZipSource(zipBytes), state, emit, {
+    zstdDecoder,
+    onInvalid: (detail) => emitShsError(emit, 'invalid-event-log', detail ?? undefined),
+  });
+}
+
+const baseName = (e          ) => e.name.slice(e.name.lastIndexOf('/') + 1);
+const isRollingPart = (e          ) => /^events_\d+_/.test(baseName(e));
+
+// The application attempts a History Server zip holds. Without an attempt ID
+// the History Server packs every attempt into one download: each single-file
+// log is its own root entry, each rolling log its own `eventlog_v2_` directory.
+// Directory entries and `appstatus` markers are not attempts.
+function countLogAttempts(entries            )         {
+  const attempts = new Set        ();
+  for (const e of entries) {
+    if (e.name.endsWith('/') || baseName(e).toLowerCase().startsWith('appstatus')) continue;
+    const dir = e.name.slice(0, e.name.indexOf('/') + 1);
+    attempts.add(dir || (isRollingPart(e) ? '' : e.name));
+  }
+  return attempts.size;
+}
+
+// The event-log entries a single-attempt History Server zip holds, in parse
+// order. A rolling log's parts sit under an `eventlog_v2_<appId>/` directory
+// entry; they are matched and reassembled by base name. A single-file log is
+// one entry at the root. Directory entries and the rolling log's `appstatus`
+// marker are skipped.
+function selectLogEntries(entries            )             {
+  const files = entries.filter((e) => !e.name.endsWith('/'));
+  if (files.some(isRollingPart)) {
+    const byBase = new Map(files.map((e) => [baseName(e), e]));
+    return reassembleRollingEntries(files.map(baseName)).map((n) => byBase.get(n) );
+  }
+  return files.filter((e) => baseName(e).toLowerCase() !== 'appstatus');
+}
+
+                               
+                                   
+                     
+                                                                            
+                                                                             
+                         
+                                             
+                                         
+                         
+                                                                             
+                      
+ 
+
+// Parses a single-entry or rolling Spark History Server zip, streaming each
+// entry out of `source` in `chunkSize` slices. One NDJSON line decoder spans
+// all entries, since a file-roll boundary need not fall on a line boundary.
+export async function parseZipArchive(
+  source           ,
+  state             ,
+  emit        ,
+  { zstdDecoder, chunkSize = 512 * 1024, onInvalid, progressEvery = 2000, reportPct = false }              ,
+)                {
+  let logEntries            ;
+  let attempts        ;
   try {
-    entries = unzipSync(zipBytes);
-  } catch {
-    emitShsError(emit, 'invalid-event-log');
+    const entries = await listZipEntries(source);
+    attempts = countLogAttempts(entries);
+    logEntries = attempts > 1 ? [] : selectLogEntries(entries);
+  } catch (e) {
+    onInvalid(`Could not read the zip archive: ${e instanceof Error ? e.message : String(e)}`);
     return;
   }
-  const allNames = Object.keys(entries);
-  const isRolling = allNames.some(n => /^events_\d+_/.test(n));
-  let names;
-  if (isRolling) {
-    try {
-      names = reassembleRollingEntries(allNames);
-    } catch {
-      emitShsError(emit, 'invalid-event-log');
-      return;
-    }
-  } else {
-    names = allNames.filter(n => n.toLowerCase() !== 'appstatus').sort(naturalCompare);
+  if (attempts > 1) {
+    onInvalid(`The zip archive holds ${attempts} application attempts. Download a single attempt, for example GET /api/v1/applications/<appId>/<attemptId>/logs.`);
+    return;
   }
-  if (names.length === 0) {
-    emitShsError(emit, 'invalid-event-log');
+  if (logEntries.length === 0) {
+    onInvalid('The zip archive contains no event log.');
     return;
   }
 
+  const totalBytes = logEntries.reduce((sum, e) => sum + e.compressedSize, 0);
+  let bytesRead = 0;
   const decoder = buildChunkDecoder();
   const joined               = [];
   let linesProcessed = 0;
-  for (const name of names) {
+  const feed = (bytes            ) => {
+    joined.length = 0;
+    const lines = decoder.decode(bytes, joined);
+    for (let i = 0, j = 0; i < lines.length; i++) {
+      dispatchLine(lines[i], state, emit, joined[j]?.index === i ? joined[j++] : undefined);
+      linesProcessed++;
+      if (linesProcessed % progressEvery === 0) {
+        emit({ type: 'progress', pct: reportPct && totalBytes ? bytesRead / totalBytes : null, linesProcessed });
+      }
+    }
+  };
+
+  for (const entry of logEntries) {
+    const entryDecoder = createEntryDecoder(entry.name, feed, zstdDecoder);
     try {
-      decodeEntry(name, entries[name], (bytes) => {
-        joined.length = 0;
-        const lines = decoder.decode(bytes, joined);
-        for (let i = 0, j = 0; i < lines.length; i++) {
-          dispatchLine(lines[i], state, emit, joined[j]?.index === i ? joined[j++] : undefined);
-          linesProcessed++;
-          if (linesProcessed % 2000 === 0) {
-            emit({ type: 'progress', pct: null, linesProcessed });
-          }
-        }
-      }, zstdDecoder);
-    } catch {
-      emitShsError(emit, 'invalid-event-log');
+      await streamZipEntry(source, entry, (chunk) => entryDecoder.push(chunk), {
+        chunkSize,
+        onRead: (bytes) => { bytesRead += bytes; },
+      });
+      await entryDecoder.end();
+    } catch (e) {
+      entryDecoder.cancel();
+      onInvalid(`Could not decompress "${entry.name}" in the zip archive: ${e instanceof Error ? e.message : String(e)}`);
       return;
     }
   }
@@ -198,7 +308,7 @@ export function decodeShsArchive(
   }
 
   if (!state.app) {
-    emitShsError(emit, 'invalid-event-log');
+    onInvalid(null);
     return;
   }
   emitParseCompletion(state, emit, linesProcessed);

package/vendor-core/shs-load.js

@@ -19,7 +19,7 @@ export const DEFAULT_IDLE_TIMEOUT_MS = envInt('SPARKFORENSICS_SHS_TIMEOUT_MS', 3
 
 function collectShsAppModel(zipBytes            )                                                        {
   return collectViaDispatch(
-    (state, emit) => decodeShsArchive(zipBytes, state, emit, nodeArchiveCodecs),
+    (state, emit, reject) => { decodeShsArchive(zipBytes, state, emit, nodeArchiveCodecs).catch(reject); },
     (msg) => {
       const m = msg                                       ;
       return mcpError(m?.code ?? 'invalid-event-log', m?.message ?? 'Failed to decode SHS archive.');

package/vendor-core/sql-stages.js

@@ -0,0 +1,11 @@
+// Shared by every scope:'sql' detector and the plan views. sql.get(id).stageIds is always empty (parser-worker
+// never populates it), so stage linkage is derived from each stage's own sqlExecutionId.
+// Minimal param shape (not DetectorStage): external callers pass Map<StageId, Stage>.
+export function stageIdsForSqlExec(
+  executionId        ,
+  stages                                                             ,
+)           {
+  const out           = [];
+  for (const s of stages.values()) if (s.sqlExecutionId === executionId) out.push(s.id);
+  return out;
+}

package/vendor-core/stage-quantiles.js

@@ -1,4 +1,5 @@
                                                             
+                                                                             
 
 // Single source of truth for the packed per-task numeric array: FIELDS (offset
 // constants), TASK_FIELD_NAMES (display labels), and the finalizeStage hot-loop
@@ -74,6 +75,8 @@ export function finalizeStage(
   const executorStats = new Map();
   const failureReasons = new Map();
   const failedTaskSamples                     = [];
+  // Keyed by the interned detail object: accumulateTask shares one per distinct failure.
+  const failureGroups = new Map                           ();
   const localityStats = new Map();
   let peakExecutionMemoryMax = 0;
 
@@ -82,6 +85,8 @@ export function finalizeStage(
     if (t.failed) {
       failedTasks++;
       if (t.reason) failureReasons.set(t.reason, (failureReasons.get(t.reason) ?? 0) + 1);
+      const failure = (t                                                     ).failure;
+      if (failure) failureGroups.set(failure, (failureGroups.get(failure) ?? 0) + 1);
       if (failedTaskSamples.length < MAX_TASK_SAMPLES) {
         failedTaskSamples.push({
           taskId: t.taskId, attemptNumber: t.attemptNumber, host: t.host, executorId: t.executorId,
@@ -125,6 +130,7 @@ export function finalizeStage(
   stage.failedTasks = failedTasks;
   stage.speculativeTasks = speculativeTasks;
   stage.taskAttempts = null; // no longer needed after finalize, freeing memory
+  stage.failureDetails = null;
 
   const arr = new Float64Array(buf);
   state.taskStore.set(stageId, arr);
@@ -164,6 +170,10 @@ export function finalizeStage(
   const failureReasonsArr = [...failureReasons.entries()].map(
     ([reason, count]) => ({ reason, count })
   );
+  // Most frequent first; ties keep first-seen order (Array.prototype.sort is stable).
+  const failureGroupsArr                     = [...failureGroups.entries()]
+    .map(([detail, count]) => ({ ...detail, count }))
+    .sort((a, b) => b.count - a.count);
   const localityStatsArr = [...localityStats.entries()].map(
     ([locality, count]) => ({ locality, count })
   );
@@ -173,6 +183,7 @@ export function finalizeStage(
     hostStats: hostStatsArr, executorStats: executorStatsArr, failureReasons: failureReasonsArr, localityStats: localityStatsArr, stragglerCount, stragglerExcessMs, longestNonStragglerMs,
     tailReplayRecoveryMs,
     failedTaskSamples,
+    failureGroups: failureGroupsArr,
     peakExecutionMemoryMax,
     taskActiveMs: computeTaskActiveMs(arr),
     peakConcurrentTasks,
@@ -187,6 +198,9 @@ export function finalizeStage(
     stageType: acc.shuffleReadBytes > 0 ? 'REDUCE' : 'MAP',
   };
   delete data.taskAttempts; // internal-only field, already nulled above; never part of the public message
+  delete data.failureDetails; // internal-only intern table, summarized by failureGroups
+  delete data.speculativeWinners; // internal-only late-TaskEnd pairing state, kept worker-side
+  delete data.lateSpeculationWaste;
 
   return { type: 'stage', data };
 }

package/vendor-core/task-failure.js

@@ -0,0 +1,151 @@
+// What actually went wrong behind a failed task attempt, read from its TaskEnd "Task End Reason".
+// Spark's `Reason` is only the end-reason tag (ExceptionFailure, ExecutorLostFailure, ...); the
+// real error lives in tag-specific fields (JsonProtocol.taskEndReasonToJson):
+//   ExceptionFailure     Class Name, Description, Full Stack Trace
+//   ExecutorLostFailure  Loss Reason (e.g. "Container killed by YARN for exceeding memory limits")
+//   FetchFailed          Message (often a whole exception string, stack included)
+//   TaskKilled           Kill Reason
+// Every text field is bounded here, at ingest, so neither parser memory nor a report grows with
+// the size of the traces Spark wrote.
+
+                                    
+                        
+                           
+                         
+                            
+                              
+ 
+
+                                                             
+                
+ 
+
+const MAX_TEXT_CHARS = 300;
+const MAX_EXCERPT_FRAMES = 8;
+const MAX_EXCERPT_LINE_CHARS = 300;
+const MAX_EXCERPT_CHARS = 2000;
+// Distinct failures kept per stage while parsing, and shown per finding.
+export const MAX_FAILURE_DETAILS_PER_STAGE = 50;
+export const MAX_FAILURE_GROUPS = 5;
+
+export const REDACTED_TEXT = '[redacted]';
+
+function truncate(s        , max        )         {
+  return s.length > max ? `${s.slice(0, max - 3)}...` : s;
+}
+
+function text(v         )                {
+  return typeof v === 'string' && v.trim().length > 0 ? v : null;
+}
+
+// First non-blank line, whitespace collapsed: a message can carry a whole stack trace after it.
+function firstLine(s               )                {
+  if (s == null) return null;
+  const line = s.split('\n').map((l) => l.trim()).find((l) => l.length > 0);
+  return line ? truncate(line.replace(/\s+/g, ' '), MAX_TEXT_CHARS) : null;
+}
+
+const PYTHON_TRACEBACK_HEAD = /(?:^|: )(?:Traceback \(most recent call last\):|An exception was thrown from the Python worker)/;
+const JAVA_FRAME = /^\s*at \S/;
+
+// A PySpark error's text is a whole Python traceback whose own error line (`ValueError: bad row`)
+// comes last, before any JVM frames: index of that line among non-blank `lines`, or -1.
+function pythonErrorLineIndex(lines          )         {
+  if (lines.length < 2 || !PYTHON_TRACEBACK_HEAD.test(lines[0].trim())) return -1;
+  let last = -1;
+  for (let i = 1; i < lines.length && !JAVA_FRAME.test(lines[i]); i++) last = i;
+  return last;
+}
+
+// The message's headline: its first line, or a Python traceback's error line.
+function messageLine(s               )                {
+  if (s == null) return null;
+  const lines = s.split('\n').filter((l) => l.trim().length > 0);
+  const pyIdx = pythonErrorLineIndex(lines);
+  return pyIdx >= 0 ? firstLine(lines[pyIdx]) : firstLine(s);
+}
+
+/** Header line, the first frames, and (when cut off) a Python traceback's error line and the last
+ * `Caused by:` line, which usually name the root cause. Bounded by frame count, line length and
+ * total length. */
+export function buildStackExcerpt(trace               )                {
+  if (trace == null) return null;
+  const lines = trace.split('\n').map((l) => l.replace(/\s+$/, '')).filter((l) => l.trim().length > 0);
+  if (lines.length === 0) return null;
+  const headCount = 1 + MAX_EXCERPT_FRAMES;
+  const kept = lines.slice(0, headCount);
+  if (lines.length > headCount) {
+    const pyIdx = pythonErrorLineIndex(lines);
+    let causeIdx = -1;
+    for (let i = lines.length - 1; i >= headCount; i--) {
+      if (lines[i].startsWith('Caused by:')) { causeIdx = i; break; }
+    }
+    if (pyIdx >= headCount) kept.push('\t...', lines[pyIdx]);
+    if (pyIdx < lines.length - 1) kept.push('\t...');
+    if (causeIdx >= 0) kept.push(...lines.slice(causeIdx, causeIdx + 2));
+  }
+  return truncate(kept.map((l) => truncate(l, MAX_EXCERPT_LINE_CHARS)).join('\n'), MAX_EXCERPT_CHARS);
+}
+
+/** Bounded failure detail for a failed attempt's end reason, or null when there is no end reason. */
+export function extractTaskFailureDetail(endReason                                     )                           {
+  if (!endReason) return null;
+  const reason = text(endReason['Reason']);
+  const rawMessage = text(endReason['Description']) ?? text(endReason['Message']) ?? text(endReason['Kill Reason']);
+  // FetchFailed has no Full Stack Trace field, but its Message is a full exception string.
+  const trace = text(endReason['Full Stack Trace'])
+    ?? (rawMessage != null && rawMessage.trim().includes('\n') ? rawMessage : null);
+  return {
+    reason,
+    className: firstLine(text(endReason['Class Name'])),
+    message: messageLine(rawMessage),
+    lossReason: firstLine(text(endReason['Loss Reason'])),
+    stackExcerpt: buildStackExcerpt(trace),
+  };
+}
+
+/** Grouping key: one group per distinct error. The excerpt is left out, so two traces of the same
+ * error share a group and the first one seen is kept. */
+export function taskFailureKey(d                   )         {
+  return JSON.stringify([d.reason, d.className, d.message, d.lossReason]);
+}
+
+/** Short name for the error: the exception class, or the end-reason tag with its loss reason. */
+export function describeTaskFailure(d                   )                {
+  if (d.className) return d.className;
+  if (d.lossReason) return d.reason ? `${d.reason}: ${d.lossReason}` : d.lossReason;
+  return d.reason;
+}
+
+const FRAME_LINE = /^\s*(?:at \S.*|\.\.\.(?: \d+ more)?)$/;
+const CLASS_HEADER = /^((?:Caused by: |Suppressed: )?[\w$]+(?:\.[\w$]+)+)(?::.*)?$/;
+
+// Keeps stack frames and exception class names only: a message (the text after "Class: ", and any
+// continuation line such as a Python traceback's `File "/path"` lines) is dropped.
+function stripExcerptMessages(excerpt        )         {
+  const kept           = [];
+  for (const line of excerpt.split('\n')) {
+    if (FRAME_LINE.test(line)) { kept.push(line); continue; }
+    const header = CLASS_HEADER.exec(line.trim());
+    if (header) kept.push(header[1]);
+  }
+  return kept.join('\n');
+}
+
+/** Redacted copy of a failure group: the message and the message text inside the stack excerpt
+ * can carry file paths and data values. Class names, frames and the loss reason are kept (hosts in
+ * a loss reason are pseudonymized by the caller's host scan like any other free text). */
+export function redactTaskFailureGroup                             (g   )    {
+  return {
+    ...g,
+    message: g.message == null ? null : REDACTED_TEXT,
+    stackExcerpt: g.stackExcerpt == null ? null : stripExcerptMessages(g.stackExcerpt),
+  };
+}
+
+/** One-line headline for a failure group, for reports and the Failed Tasks widget. */
+export function formatTaskFailureHeadline(d                   )         {
+  const name = d.className ?? d.reason ?? 'Unknown failure';
+  const detail = [d.message, d.lossReason].filter((s)              => s != null).join(' · ');
+  return detail ? `${name}: ${detail}` : name;
+}