sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/mcp-tools.js

@@ -6,16 +6,21 @@ import { collectRun } from './cli/collect-run.js';
 import { deriveEvidenceAvailability } from './evidence-availability.js';
 import { resolveFromShs, DEFAULT_MAX_ARCHIVE_BYTES, DEFAULT_IDLE_TIMEOUT_MS } from './shs-load.js';
 import { mcpError } from './mcp-error.js';
-import { buildEvidenceReport, toFindingsFilter,                                                                                        } from './evidence-report.js';
-import { redactAppIdentity, redactComparison } from './redact.js';
+import { buildEvidenceReport, toFindingsFilter,                                                                                                               } from './evidence-report.js';
+import { redactComparison } from './redact.js';
 import { computeWallClock } from './wall-clock.js';
 import { analyze } from './analyzer.js';
 import { buildComparison, renderComparisonMarkdown,                        } from './run-comparison.js';
+import { comparisonVerdict,                            } from './comparison-verdict.js';
 import { evaluateBudgets,                                       } from './cli/budgets.js';
 import { FINDING_NAMES, titleCase } from './finding-names.js';
-import { docAnchorForType, tuningDocSlugForAnchor, pageForAnchor } from './docs-config.js';
+import { docAnchorForType } from './detector-docs.js';
+import { DETECTORS,                         } from './detectors.js';
+import { tunedDetectors } from './threshold-overrides.js';
+import { tuningDocSlugForAnchor, pageForAnchor } from './docs-config.js';
 import { typeTag } from './format-utils.js';
-                                                                  
+                                                                                   
+                                               
 
                                                                                                      
 // RunRef uses a nested `source` key, matching every real call site (resolveOrCreateRun's
@@ -33,13 +38,22 @@ import { typeTag } from './format-utils.js';
                                                                                                    
                                                                 
                        
+                                                                                      
+                     
+                    
+                               
+                                      
+                                                                                               
+                     
  
 // compareRuns returns a smaller MCP-facing projection of CompareRunsResult
-// (runIdA/runIdB/findingsDelta/metricDeltas/confidence/reason/matchedCoverage), not the full raw
-// shape (no baselineLabel/stageSkew/baseStages/candStages).
+// (runIdA/runIdB/verdict/findingsDelta/metricDeltas/confidence/reason/matchedCoverage), not the full
+// raw shape (no baselineLabel/stageSkew/baseStages/candStages/jobOutcomes).
                                        
                  
                  
+                                                                               
+                                 
                                                
                                              
                                               
@@ -169,20 +183,29 @@ export async function resolveOrCreateRun(
   return resolution;
 }
 
+// `thresholds` on every analyzing tool below: the server's --thresholds overrides, fixed for the
+// process by createMcpServer(). A client can't set them per call; the dashboard never has them.
 export function diagnoseRun(runId        , opts    
                                                                                                           
-                                                           
+                                                                                            
  )   
-                                                              
-                                                                       
+                                                                                                      
+                                                                                                         
+                                                    
                                                                                                                 {
   const appModel = getCachedAppModel(runId);
   const findingsFilter = toFindingsFilter(opts?.impactBand, opts?.type, opts?.stageId);
-  const { json, markdown } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: opts?.markdown, findingsFilter });
+  const { json, markdown } = buildEvidenceReport(appModel, {
+    redact: opts?.redact, markdown: opts?.markdown, findingsFilter, thresholds: opts?.thresholds,
+  });
   const include = opts?.include ?? [];
+  const tuned = json.summary.tunedThresholds;
   return {
-    runId, findings: json.findings, recommendations: json.recommendations, cleanChecks: json.cleanChecks,
+    runId, verdict: json.verdict, findings: json.findings, recommendations: json.recommendations, cleanChecks: json.cleanChecks,
+    notRunChecks: json.notRunChecks,
     runComplete: appModel.app?.endTime != null,
+    // Top level too, so a client that never asks for `summary` still sees the run was tuned.
+    ...(tuned ? { tunedThresholds: tuned } : {}),
     ...(include.includes('summary') ? { summary: json.summary } : {}),
     ...(include.includes('evidenceAvailability') ? { evidenceAvailability: json.evidenceAvailability } : {}),
     ...(include.includes('detectors') ? { detectors: json.detectors } : {}),
@@ -211,17 +234,26 @@ function extractDocTitle(markdown        )         {
                                                                        
  
 
+// A finding type documents itself; a detector-level type that never appears on a finding
+// (broadcastSizing) documents the types its entry emits.
+function documentedTypes(type        )                    {
+  if (FINDING_NAMES[type] !== undefined) return [type];
+  return DETECTORS.find((entry) => entry.type === type)?.emits ?? [];
+}
+
 /** Detection + tuning reference documentation for one finding `type`, independent of any run
- * (documentation is a property of the type: a client fetches it once per type and caches it). */
+ * (documentation is a property of the type: a client fetches it once per type and caches it).
+ * A detector-level `type` resolves to the documentation of the finding types its entry emits. */
 export function getFindingDocumentation(type        )                       {
-  const label = FINDING_NAMES[type];
-  if (label === undefined) throw mcpError('invalid-type', `Unknown finding type: ${type}`);
+  const types = documentedTypes(type);
+  const [docType] = types;
+  if (docType === undefined) throw mcpError('invalid-type', `Unknown finding type: ${type}`);
 
-  const tag = typeTag(type);
+  const tag = typeTag(docType);
   const detectionContent = readFileSync(join(DOCS_CONTENT_DIR, 'detection', `${tag.toLowerCase()}.md`), 'utf8');
   const detectionDoc = { tag, title: extractDocTitle(detectionContent), content: detectionContent };
 
-  const anchor = docAnchorForType(type);
+  const anchor = docAnchorForType(docType);
   const slug = anchor ? tuningDocSlugForAnchor(anchor) : null;
   const tuningPath = slug ? join(DOCS_CONTENT_DIR, 'tuning', `${slug}.md`) : null;
   let tuningDoc                                    = null;
@@ -235,7 +267,8 @@ export function getFindingDocumentation(type        )                       {
     if (entry) tuningDoc = { anchor, title: entry.title, content: readNavEntryContent(entry) };
   }
 
-  return { type, name: titleCase(label), detectionDoc, tuningDoc };
+  const name = types.map((t) => titleCase(FINDING_NAMES[t] ?? t)).join(' / ');
+  return { type, name, detectionDoc, tuningDoc };
 }
 
 const CHAPTERS_NAV_FILE = join(DOCS_CONTENT_DIR, 'chapters', 'nav-index.json');
@@ -266,10 +299,10 @@ export function getReferenceDoc(anchor        )               {
 }
 
 export function getFindingEvidence(
-  runId        , findingId        , opts                       ,
+  runId        , findingId        , opts                                                        ,
 )                                         {
   const appModel = getCachedAppModel(runId);
-  const { json } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: false });
+  const { json } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: false, thresholds: opts?.thresholds });
   const finding = json.findings.find((f) => f.id === findingId);
   if (!finding) throw mcpError('finding-not-found', `No finding ${findingId} on run ${runId}.`);
   return { runId, finding };
@@ -279,15 +312,19 @@ function hasCompleteInterval(app                     )          {
   return Number.isFinite(app?.startTime) && Number.isFinite(app?.endTime) && (app?.endTime ?? 0) > (app?.startTime ?? 0);
 }
 
-export function getRunSummary(runId        , opts                       )             {
+export function getRunSummary(runId        , opts                                                        )             {
   const appModel = getCachedAppModel(runId);
   const { app, stages, jobs, sql, executors } = appModel;
   const durationMs = hasCompleteInterval(app) ? computeWallClock(app, stages).total : null;
-  // No buildEvidenceReport call here to redact, so reuse redact.ts's app-id + host-token
-  // pseudonymization directly. Passing name/sparkVersion (not just id) matters: app.name is free
-  // text and can itself carry a host/IP token.
+  // The failure reason needs the stageFailed findings, so this reads the (cached) evidence report.
+  // With redact, the app identity comes from that same redacted report, so a host token in the app
+  // name and in Spark's failure reason get the same pseudonym.
+  const { summary } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: false, thresholds: opts?.thresholds }).json;
+  const { outcome } = summary;
   const rawApp = { id: app?.id ?? null, name: app?.name ?? null, sparkVersion: app?.sparkVersion ?? null };
-  const redactedApp = opts?.redact ? redactAppIdentity(rawApp) : rawApp;
+  const redactedApp = opts?.redact
+    ? { id: summary.app.id ?? null, name: summary.app.name ?? null, sparkVersion: summary.app.sparkVersion ?? null }
+    : rawApp;
   return {
     runId,
     app: redactedApp,
@@ -297,27 +334,31 @@ export function getRunSummary(runId        , opts                       )
     executorCount: { added: executors.added.length, removed: executors.removed.length },
     durationMs,
     runComplete: app?.endTime != null,
+    ...outcome,
+    runShape: summary.runShape,
   };
 }
 
 // Shared by compareRuns and evaluateBudgetsForRun: both need a resolved run's finding catalog
 // (same analyze() call shape) first.
-async function resolveAndAnalyze(ref        )                                                                     {
+async function resolveAndAnalyze(
+  ref        , thresholds                                ,
+)                                                                     {
   const { runId, appModel } = await resolveOrCreateRun(ref);
   const catalog = analyze(
     appModel.app, appModel.stages, appModel.executors.added, appModel.executors.removed,
-    appModel.jobs, appModel.sql, appModel.runAggregates,
+    appModel.jobs, appModel.sql, appModel.runAggregates, { thresholds },
   );
   return { runId, appModel, catalog };
 }
 
 export async function compareRuns(
-  a        , b        , opts                                           ,
-)                                                        {
+  a        , b        , opts                                                                            ,
+)                                                                                                           {
   const [
     { runId: runIdA, appModel: appModelA, catalog: catalogA },
     { runId: runIdB, appModel: appModelB, catalog: catalogB },
-  ] = await Promise.all([resolveAndAnalyze(a), resolveAndAnalyze(b)]);
+  ] = await Promise.all([resolveAndAnalyze(a, opts?.thresholds), resolveAndAnalyze(b, opts?.thresholds)]);
 
   // buildComparison's captureSnapshot uses an empty taskDataCache: that arg only feeds the
   // interactive stage-detail drill-down, which none of compare/matchStages/metricDeltas/findingsDelta
@@ -330,44 +371,63 @@ export async function compareRuns(
   // Stage names throughout `built` carry raw Spark stage text, which can embed a host/IP token as
   // free text, the same residual redactReport() already scrubs from the evidence report.
   const result = opts?.redact ? redactComparison(built) : built;
+  const verdict = comparisonVerdict(result);
 
   return {
     runIdA,
     runIdB,
+    verdict,
     findingsDelta: result.findings,
     metricDeltas: result.metrics,
     confidence: result.confidence,
     reason: result.reason,
     matchedCoverage: result.matchedCoverage,
-    ...(opts?.markdown ? { markdown: renderComparisonMarkdown(result) } : {}),
+    ...tunedField(opts?.thresholds),
+    ...(opts?.markdown ? { markdown: renderComparisonMarkdown(result, verdict, tunedDetectors(opts?.thresholds)) } : {}),
   };
 }
 
+// Both runs of a comparison use the same overrides, so a tuned delta compares like with like.
+function tunedField(thresholds                                )                                                        {
+  const tuned = tunedDetectors(thresholds);
+  return tuned ? { tunedThresholds: tuned } : {};
+}
+
 export async function evaluateBudgetsForRun(
   primary        ,
   budgets               ,
   secondary         ,
-)                                                                                                {
+  opts                                      ,
+)           
+                                                                                   
+                                                    
+   {
   // Mirrors the CLI's --regression-metric/--max-regression-pct pairing guard: unlike the CLI, this
   // tool never defaults regressionMetric, so seeing it set here means the caller asked for a
   // regression check and forgot the threshold, evaluateBudgets() would otherwise skip it silently.
   if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
     throw mcpError('access-or-upstream-failure', 'regressionMetric requires maxRegressionPct.');
   }
-  const [{ runId, appModel, catalog }, second] = await Promise.all([
-    resolveAndAnalyze(primary),
-    secondary ? resolveAndAnalyze(secondary) : Promise.resolve(undefined),
+  const [first, second] = await Promise.all([
+    resolveAndAnalyze(primary, opts?.thresholds),
+    secondary ? resolveAndAnalyze(secondary, opts?.thresholds) : Promise.resolve(undefined),
   ]);
 
-  let comparison                               ;
-  if (second) {
-    comparison = buildComparison(
-      { label: runId, appModel, catalog },
-      { label: second.runId, appModel: second.appModel, catalog: second.catalog },
-    );
-  }
-
-  const { results, violated, inconclusive } = evaluateBudgets({ appModel, catalog, budgets, comparison });
-
-  return { runId, results, violated, inconclusive };
+  // Same roles as the CLI's positional run + --baseline: with a second run, `primary` is the
+  // baseline and `secondary` the candidate, and absolute budgets plus the run-complete check
+  // apply to the candidate.
+  const candidate = second ?? first;
+  const baseline = second ? first : undefined;
+  const comparison                                = baseline
+    ? buildComparison(
+      { label: baseline.runId, appModel: baseline.appModel, catalog: baseline.catalog },
+      { label: candidate.runId, appModel: candidate.appModel, catalog: candidate.catalog },
+    )
+    : undefined;
+
+  const { results, violated, inconclusive } = evaluateBudgets({
+    appModel: candidate.appModel, catalog: candidate.catalog, budgets, comparison, thresholds: opts?.thresholds,
+  });
+
+  return { runId: first.runId, results, violated, inconclusive, ...tunedField(opts?.thresholds) };
 }

package/vendor-core/model-assembler.js

@@ -63,6 +63,18 @@ export function createModelCallbacks(
         if (stage) stage.executorMetrics = execMetrics;
       }
     },
+    // Patch speculation totals that grew after StageCompleted: Spark kills a losing speculative
+    // copy only once its stage finishes. `data` is Map<stageId, { speculationWasteMs, speculationWastedAttempts }>.
+    onStageSpeculationWaste(data         ) {
+      const totalsByStage = data                                                                                  ;
+      for (const [stageId, totals] of totalsByStage) {
+        const stage = appModel.stages.get(stageId);
+        if (stage) {
+          stage.speculationWasteMs = totals.speculationWasteMs;
+          stage.speculationWastedAttempts = totals.speculationWastedAttempts;
+        }
+      }
+    },
     onDone, onError,
   };
 }

package/vendor-core/occupancy.js

@@ -1,7 +1,7 @@
 // Occupancy-weighted wall-clock attribution for per-finding impact: every
 // stage's wall-clock claim is apportioned purely from its own observed window
 // and how much it overlapped with other stages. No graph, no parentIds traversal.
-import { mergeIntervals } from './wall-clock.js';
+import { mergeIntervals } from './intervals.js';
 
                                  
              

package/vendor-core/parser-worker.js

@@ -4,14 +4,15 @@ import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
 import { createSnappyBlockDecoder } from './snappy-block.js';
 import { createState, dispatchLine, buildChunkDecoder, emitParseCompletion,                                   } from './event-handlers.js';
 import { TASK_FIELD_NAMES } from './stage-quantiles.js';
-import { runParseFromUrl, sniffCodec } from './shs-fetch.js';
+import { runParseFromUrl, sniffCodec, parseZipArchive } from './shs-fetch.js';
+import { isZip } from './zip-archive.js';
 import { createWorkerZstdDecoders } from './zstd-worker-client.js';
 
 export {
   buildChunkDecoder, createState, normalizeSparkProperties, parseSparkMemoryMB, extractResources,
   accumulateTask, resolvePlanTree, startApplication, updateEnvironment, startJob, endJob, submitStage,
   mergeStageRddInfo, recordStageExecutorMetrics, startSqlExecution, endSqlExecution,
-  applyDriverAccumUpdates, addExecutor, removeExecutor, processEvent, dispatchLine,
+  applyDriverAccumUpdates, addExecutor, removeExecutor, recordBlockUpdate, processEvent, dispatchLine,
   collectStageExecutorMetrics,
 } from './event-handlers.js';
 
@@ -35,6 +36,8 @@ const MIN_PROGRESS_STEPS = 100;
 const PROGRESS_EMIT_LINES = 300;
 
                                      
+const NOT_AN_EVENT_LOG = 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.';
+
 // zstdDecoder replaces the vendored fzstd for zstd input; the Node CLI/MCP path passes
 // cli/native-zstd.ts's native-zlib decoder, which a browser bundle can't import.
                                                                                        
@@ -140,6 +143,20 @@ export async function runParse(
     return;
   }
 
+  // A Spark History Server download (the UI's download link, or GET
+  // /api/v1/applications/<id>/logs) is a zip holding the log file, or a
+  // rolling log's parts: unwrap it through the same path the SHS fetch uses.
+  if (isZip(new Uint8Array(await file.slice(0, Math.min(4, file.size)).arrayBuffer()))) {
+    await parseZipArchive(file, state, emit, {
+      zstdDecoder,
+      chunkSize,
+      progressEvery: PROGRESS_EMIT_LINES,
+      reportPct: true,
+      onInvalid: (detail) => emit({ type: 'error', message: detail ?? NOT_AN_EVENT_LOG }),
+    });
+    return;
+  }
+
   const decoder = buildChunkDecoder();
   const joined               = [];
   let linesProcessed = 0;
@@ -168,7 +185,7 @@ export async function runParse(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
+    emit({ type: 'error', message: NOT_AN_EVENT_LOG });
     return;
   }
 
@@ -227,7 +244,7 @@ export async function runParseFiles(
   }
 
   if (!state.app) {
-    emit({ type: 'error', message: 'Not a Spark event log: no application-start event found. Choose a Spark event log file, or check the docs for supported formats.' });
+    emit({ type: 'error', message: NOT_AN_EVENT_LOG });
     return;
   }
 
@@ -242,7 +259,7 @@ if (isWorker) {
   let workerState                     = null;
   // Dropped zstd files decompress in a second worker, overlapping with parsing here. The
   // `new Worker(new URL(...))` stays inline for Vite's worker detection (see ingest.ts). The SHS
-  // path (runParseFromUrl) decodes whole zip entries synchronously and keeps in-thread fzstd.
+  // path (runParseFromUrl) keeps in-thread fzstd.
   const zstdDecoder = createWorkerZstdDecoders(
     () => new Worker(new URL('./zstd-worker.js', import.meta.url), { type: 'module' }),
     (onChunk) => new (ZstdDecompress                                   )(onChunk),

package/vendor-core/plan-graph-model.js

@@ -15,7 +15,7 @@ import {
   formatPlanMetricValue,
 } from './plan-node-detail.js';
 import { computeSegments, mapSegmentsToStagesForDisplay } from './plan-duration-attribution.js';
-import { stageIdsForSqlExec } from './detectors.js';
+import { stageIdsForSqlExec } from './sql-stages.js';
              
                                                                                                        
                     
@@ -121,7 +121,8 @@ export function buildPlanGraphModel(
   const findingsByNodeId = new Map                   ();
   if (sqlExecutionId != null) {
     for (const finding of findings) {
-      if (finding.executionId !== sqlExecutionId) continue;
+      if (!('executionId' in finding) || finding.executionId !== sqlExecutionId) continue;
+      // `?? []`: a hand-built or foreign finding may still lack the field the type promises.
       for (const nodeId of finding.planNodeIds ?? []) {
         const list = findingsByNodeId.get(nodeId);
         if (list) list.push(finding);

package/vendor-core/plan-node-detail.js

@@ -5,7 +5,7 @@
 
 import { pathBasename, formatBytes, formatDuration } from './format-utils.js';
 import { attributeStageDurationToPlan, attributeStageDurationToPlanInclusive } from './plan-duration-attribution.js';
-import { stageIdsForSqlExec } from './detectors.js';
+import { stageIdsForSqlExec } from './sql-stages.js';
                                                                                           
 
 const BOILERPLATE_PREFIXES = ['serializefromobject', 'deserializetoobject', 'mapelements',

package/vendor-core/recommendation-rollup.js

@@ -1,7 +1,7 @@
 // Explicit .ts extensions: plain Node's ESM resolver (the runtime CLI/MCP path
 // runs under) requires the exact specifier, unlike a bundler.
-import { mergeIntervals } from './wall-clock.js';
-import { worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
+import { mergeIntervals } from './intervals.js';
+import { formatWallClockRange, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
                                                                     
 
                          
@@ -48,7 +48,7 @@ function groupByType(findings           )                         {
 }
 
 function stageIdsOf(finding         )           {
-  if (finding.stageIds) return finding.stageIds;
+  if ('stageIds' in finding) return finding.stageIds;
   if (finding.stageId != null) return [finding.stageId];
   return [];
 }
@@ -124,6 +124,31 @@ export function buildRecommendationRollup(
   });
 }
 
+/** A group's trailing figure on the Findings board, and its tooltip. `time`
+ * and `resource` both lead with the "×N" finding count (the "worth
+ * expanding" signal); `count` skips it since the impact-band tally already
+ * implies N. The row stays terse ("×2 · 476ms recoverable") to fit a dense
+ * right-aligned column; the title spells the shorthand out. */
+export function rollupGroupStat(group             )                                      {
+  if (group.kind === 'time') {
+    const recoverable = formatWallClockRange(group.recoverableMsHigh, group.recoverableMsHigh);
+    return {
+      stat: `×${group.findingCount} · ${recoverable} recoverable`,
+      statTitle: `${group.findingCount} findings of this type; up to ${recoverable} of run time could be recovered by fixing them`,
+    };
+  }
+  if (group.kind === 'resource') {
+    return {
+      stat: `×${group.findingCount} · resource-cost projection`,
+      statTitle: `${group.findingCount} findings of this type; a resource-cost estimate (not run time) is projected for fixing them`,
+    };
+  }
+  return {
+    stat: Object.entries(group.byImpactBand).map(([impactBand, count]) => `${count} ${impactBand}`).join(', '),
+    statTitle: `${group.findingCount} findings of this type, by impact`,
+  };
+}
+
 /** True when a finding is real evidence of an issue, as opposed to a mere
  * evidence-unavailable caveat. Shared by `isEligible` below and by anything
  * else that decides whether a REGISTRY type has "something to show" (an
@@ -150,6 +175,10 @@ export function isEligible(finding         )          {
   // `isRealFinding`, this exclusion doesn't apply beyond the rollup: an
   // incompleteRun finding still backs its own ordinary active widget card.
   if (finding.type === 'incompleteRun') return false;
+  // A missing-evidence caveat (memoryUtilization's is already excluded by isRealFinding;
+  // cacheUtilization's storageUnobserved keeps its widget card, so a run with no block updates
+  // never lists Cache Storage as a passed check) is never a fix to rank.
+  if ('dataUnavailable' in finding && finding.dataUnavailable) return false;
   return isRealFinding(finding);
 }
 
@@ -191,3 +220,34 @@ export function rankFindings(findings           )            {
     return (IMPACT_BAND_ORDER[a.impactBand] ?? 9) - (IMPACT_BAND_ORDER[b.impactBand] ?? 9);
   });
 }
+
+/** A board group ready to render: members ranked representative first, the representative's
+ * band (the heading the group sits under) and the group's trailing figure. */
+                                    
+                            
+               
+                            
+                   
+                     
+               
+                    
+ 
+
+/** The Findings board's groups over `eligible`, in fix-first order. The run's interpretation
+ * carries this for every eligible finding; the board recomputes it only over a filtered subset. */
+export function rankedRollup(
+  eligible           ,
+  stages                                                             ,
+)                      {
+  return buildRecommendationRollup(eligible, stages).map((group) => {
+    const members = rankFindings(group.findings);
+    return {
+      kind: group.kind,
+      type: group.type,
+      unit: group.kind === 'resource' ? group.unit : null,
+      band: members[0].impactBand,
+      members,
+      ...rollupGroupStat(group),
+    };
+  });
+}