sparkforensics-mcp 0.3.0 → 0.4.0

package/vendor-core/impact-model.js

@@ -0,0 +1,175 @@
+// The waste models every detector entry's estimate() builds its ImpactEstimate from: the assumed
+// throughputs, the per-stage measurements behind them and the occupancy clip wrappers. Each
+// finding type's own composition of these lives on its DETECTORS entry, next to its detect().
+                                                                                              
+import { nsToMs } from './format-utils.js';
+import {
+  estimateSingleStage, estimateMultiStage,
+                                                                                
+} from './occupancy.js';
+
+/** What every estimate() is handed: built once per analyze(), and the same object detect() gates
+ * its runtime floors against (DetectorCtx.impact), so a floor and the savings displayed for it
+ * read one occupancy sweep. */
+                              
+                             
+                                             
+                                                                                 
+                     
+ 
+
+// Assumed shuffle-network throughput per executor link, ~1 Gbps. Starting assumption, unvalidated.
+export const SHUFFLE_THROUGHPUT_BPS = 125_000_000;
+// Assumed disk I/O throughput per executor for spilled data, ~200 MB/s (conservative HDD/SSD blend).
+export const SPILL_IO_THROUGHPUT_BPS = 200_000_000;
+
+// A stage's own per-task overhead: task wall time (launch to finish, summed per executor by
+// finalizeStage) minus executorRunTime, i.e. deserialization, result serialization and the
+// launch round trip that coalescing tasks removes. `concurrency` is the stage's achieved task
+// concurrency (task time / stage duration). Null when the stage has no such data or no overhead.
+export function measuredTaskOverhead(stage       )                                                    {
+  const executorStats = Array.isArray(stage.executorStats)
+    ? (stage.executorStats                                ) : [];
+  const taskTimeMs = executorStats.reduce((sum, e) => sum + (e.totalDuration ?? 0), 0);
+  const taskCount = stage.taskCount ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  const overheadMs = taskTimeMs - (stage.executorRunTime ?? 0);
+  if (taskTimeMs <= 0 || taskCount <= 0 || durationMs <= 0 || overheadMs <= 0) return null;
+  return { perTaskMs: overheadMs / taskCount, concurrency: taskTimeMs / durationMs };
+}
+
+// Both constants above are single-device figures (one NIC, one local disk). A stage's shuffle
+// reads and spills are spread over every executor that ran its tasks, each moving its own share
+// in parallel, so the stage's aggregate bandwidth scales with that executor count. Dividing a
+// stage-wide byte total by one device's bandwidth modeled the whole cluster as one link: on 14
+// real logs that claimed up to 1,870s of shuffle time on stages whose tasks measured ~0s of
+// shuffle fetch wait (222 of 284 shuffle findings). No executor data falls back to one device.
+export function stageIoParallelism(stage       )         {
+  const executors = Array.isArray(stage.executorStats) ? stage.executorStats.length : 0;
+  return Math.max(1, executors);
+}
+
+// Wall-clock the stage's tasks spent blocked fetching shuffle blocks: fetchWaitTime is a
+// cross-task sum like executorRunTime, so dividing by the stage's average concurrency converts it
+// (the gc estimate's conversion). Null when the stage has no run time or duration to convert with.
+// On 14 real logs the link model claimed 2780s over 284 shuffle findings, 256s once capped at
+// this, with about zero fetch wait on 5 of the 10 non-info ones: reads that overlapped compute
+// stalled nothing.
+export function fetchWaitWallClockMs(stage       )                {
+  const fetchWaitMs = stage.fetchWaitTime;
+  const runTimeMs = stage.executorRunTime ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  if (typeof fetchWaitMs !== 'number' || runTimeMs <= 0 || durationMs <= 0) return null;
+  return fetchWaitMs / (runTimeMs / durationMs);
+}
+
+// Wall-clock a stage's wasted (retried) attempts cost it. Each one delayed only its own task, and
+// attempts of different tasks ran side by side: one lost executor fails every task it was running
+// at once (4 wasted attempts of 36.6s each, all first attempts, on a real stage that ran 41 tasks
+// at once, claimed as 146.6s). So the stage lost at most its longest retry chain, the most attempts
+// one task wasted (its highest attempt number + 1) times the mean wasted attempt, or their summed
+// time spread over its slots, whichever is larger. Without a sample of every wasted attempt
+// (retryTaskSamples is capped), the chain isn't known: the summed time.
+export function retryWallClockMs(stage       )         {
+  const totalMs = (stage.retryWasteMs                      ) ?? 0;
+  const attempts = (stage.wastedAttempts                      ) ?? 0;
+  const samples = Array.isArray(stage.retryTaskSamples)
+    ? (stage.retryTaskSamples                                ) : [];
+  if (totalMs <= 0 || attempts <= 0 || samples.length < attempts) return totalMs;
+  const chain = samples.reduce((longest, s) => Math.max(longest, (s.attemptNumber ?? 0) + 1), 1);
+  const slots = Math.max(1, stage.peakConcurrentTasks ?? 1);
+  return Math.min(totalMs, Math.max((chain * totalMs) / attempts, totalMs / slots));
+}
+// Spark's classic recommended shuffle partition size.
+export const IDEAL_BYTES_PER_PARTITION_TASK = 128 * 1024 * 1024;
+// Assumed per-task scheduling/launch overhead: the fallback when a stage lacks the per-executor
+// task-time sums measuredTaskOverhead needs.
+export const TASK_SCHEDULING_OVERHEAD_MS = 50;
+// Assumed per-file open latency (small-file overhead).
+export const FILE_OPEN_OVERHEAD_MS = 10;
+// Assumed broadcast-transfer bandwidth, shared with overBroadcast/underBroadcast.
+export const BROADCAST_BANDWIDTH_BPS = 125_000_000;
+// Assumed per-non-local-task network-fetch penalty, reported as extra core-time.
+export const NETWORK_FETCH_PENALTY_MS = 20;
+// Assumed executor JVM+container startup overhead.
+export const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
+// Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
+export const RE_READ_THROUGHPUT_BPS = 125_000_000;
+
+// Below this share of executorRunTime spent on CPU, a stage's tasks were idle, waiting on something
+// outside Spark: on 14 real logs (2026-09-23) every non-Python stage under 1% was a JDBC read, a
+// file listing or a Delta log read, while file writes, which more partitions do parallelize,
+// start at 2%.
+const IDLE_CPU_SHARE_MAX = 0.01;
+
+// True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
+// the share can't be trusted: no CPU time recorded (older Spark logs omit the metric), or Python
+// code run through PythonRDD, whose worker-process CPU executorCpuTime (the JVM task thread's)
+// never counts (such stages read 0.1% on the same logs while computing).
+export function tasksMostlyIdle(stage       )          {
+  const runMs = stage.executorRunTime ?? 0;
+  const cpuMs = nsToMs(stage.executorCpuTime ?? 0);
+  if (runMs <= 0 || cpuMs <= 0) return false;
+  if (/PythonRDD/.test(stage.name ?? '') || /org\.apache\.spark\.api\.python\./.test(stage.details ?? '')) return false;
+  return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
+}
+
+// skew, straggler and stageSlowness claim time off the stage's longest task itself, so the
+// occupancy clip must not floor them at that same task (see estimateSingleStage).
+export const TAIL_CLAIM                             = { shortensLongestTask: true };
+
+// No quantifiable magnitude -> 'informational'; a rawWaste figure with no stage window ->
+// 'resourceOnly'. Never a fake {low:0, high:0}: wallClock is null in both cases.
+export function costOnly(estimateMethod                      , rawWaste                 )                 {
+  return rawWaste
+    ? { basis: 'resourceOnly', wallClock: null, estimateMethod, rawWaste }
+    : { basis: 'informational', wallClock: null, estimateMethod };
+}
+
+/** The estimate() of an entry whose findings have no waste model at all. */
+export function noWasteModel()                 {
+  return costOnly('none');
+}
+
+export function singleStageImpact(
+  wasteMs        ,
+  stageId        ,
+  ctx             ,
+  estimateMethod                      ,
+  rawWaste                 ,
+  opts                             ,
+)                 {
+  const est = estimateSingleStage(wasteMs, stageId, ctx.stages                                          , ctx.occupancy, opts);
+  if (est) return { basis: est.basis, wallClock: est.wallClock, estimateMethod, rawWaste };
+  return costOnly(estimateMethod, rawWaste); // stage excluded from the sweep (duration <= 0)
+}
+
+/** estimateMultiStage over the finding's own stages, or null when every one was excluded from
+ * the sweep. */
+export function multiStageImpact(
+  stageIds          ,
+  wasteMsByStage                     ,
+  ctx             ,
+  estimateMethod                      ,
+  rawWaste                 ,
+)                        {
+  const est = estimateMultiStage(stageIds, wasteMsByStage, ctx.stages                                          , ctx.occupancy);
+  return est ? { basis: est.basis, wallClock: est.wallClock, estimateMethod, rawWaste } : null;
+}
+
+export function stageMappableWasteOrCostOnly(
+  wasteMs        ,
+  stageIds                      ,
+  ctx             ,
+)                 {
+  const rawWaste                             = wasteMs > 0 ? { value: wasteMs, unit: 'ms' } : undefined;
+  if (!stageIds || stageIds.length === 0) {
+    return costOnly('modeled', rawWaste);
+  }
+  // One waste event spread over a span of stages, not N independent wastes: apportion evenly so
+  // estimateMultiStage's union cap doesn't absorb the same amount claimed once per stage.
+  const perStageWasteMs = wasteMs / stageIds.length;
+  const wasteMsByStage = new Map(stageIds.map((id) => [id, perStageWasteMs]));
+  // null: every stage excluded from the sweep
+  return multiStageImpact(stageIds, wasteMsByStage, ctx, 'modeled', rawWaste) ?? costOnly('modeled', rawWaste);
+}

package/vendor-core/ingest.js

@@ -13,6 +13,7 @@
                                   
                                             
                                                    
+                                                    
                                    
                                     
  
@@ -37,6 +38,7 @@ export function routeMessage(
     case 'job':      handlers.onJob?.(data.data); break;
     case 'runAggregates': handlers.onRunAggregates?.(data.data); break;
     case 'stageExecutorMetrics': handlers.onStageExecutorMetrics?.(data.data); break;
+    case 'stageSpeculationWaste': handlers.onStageSpeculationWaste?.(data.data); break;
     case 'done': {
       const { skippedLines } = data;
       handlers.onDone?.({ skippedLines });

package/vendor-core/intervals.js

@@ -0,0 +1,13 @@
+// Interval union shared by the wall-clock breakdown and the savings rollups.
+export function mergeIntervals(intervals                    )                     {
+  if (intervals.length === 0) return [];
+  const sorted = [...intervals].sort((a, b) => a[0] - b[0]);
+  const out                     = [[sorted[0][0], sorted[0][1]]];
+  for (let i = 1; i < sorted.length; i++) {
+    const last = out[out.length - 1];
+    const cur = sorted[i];
+    if (cur[0] <= last[1]) last[1] = Math.max(last[1], cur[1]);
+    else out.push([cur[0], cur[1]]);
+  }
+  return out;
+}

package/vendor-core/list-runs.js

@@ -82,9 +82,8 @@ export function applyFiltersAndCap(entries                , filters
   return filters.redact ? { ...capped, runs: redactRunListEntries(capped.runs) } : capped;
 }
 
-// Local counterpart to redact.ts's redactAppIdentity, sized for a listing of many distinct apps
-// rather than the single app that tool works over: redactAppIdentity has no way to keep two
-// entries sharing an appId in sync, so this builds its own stable per-distinct-appId pseudonym map
+// Listing-wide app identity redaction: redact.ts's redactors each work over one run, with no way
+// to keep two listing entries sharing an appId in sync, so this builds its own stable per-distinct-appId pseudonym map
 // (numeric-aware sort, same scheme as redact.ts's buildMap) across the whole result set, two
 // attempts of the same app redact to the same identity, and applies it to every identity-bearing
 // field: appId, name (a listing's human-readable name is as identifying as the id itself), and the

package/vendor-core/load-vendored.js

@@ -1,16 +1,81 @@
 // Plain JS: scripts/vendor-core.mjs copies it byte-for-byte into vendor-core/, so this file exists
 // at both vendor-core/load-vendored.js (published) and core/src/load-vendored.js (dev). Each
-// package's bin bootstraps by locating this file first, then uses the exports below for every other
-// core module, so the resolution logic lives in one place instead of per entry point.
-import { existsSync } from 'node:fs';
-import { join } from 'node:path';
+// package's bin bootstraps by locating this file first (the core/src copy when it exists, so the
+// staleness check below is always current code), then uses the exports below for every other core
+// module, so the resolution logic lives in one place instead of per entry point.
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
 import { pathToFileURL } from 'node:url';
 
+// Written into vendor-core/ by scripts/vendor-core.mjs: the coreSourceHash of the core/src it was
+// built from, so a monorepo run can tell a leftover vendored copy from a current one.
+export const SOURCE_HASH_FILE = 'core-source-hash.txt';
+
+// docs-content/ is generated from a pinned upstream commit, not analysis code.
+const UNHASHED_TOP_LEVEL = new Set(['docs-content']);
+
+/** A content hash of every analysis source file under core/src (paths and bytes, sorted). */
+export function coreSourceHash(coreSrcDir) {
+  const hash = createHash('sha256');
+  const walk = (dir) => {
+    const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+      if (dir === coreSrcDir && UNHASHED_TOP_LEVEL.has(entry.name)) continue;
+      const path = join(dir, entry.name);
+      if (entry.isDirectory()) walk(path);
+      else if (entry.isFile()) hash.update(`${relative(coreSrcDir, path)}\0`).update(readFileSync(path)).update('\0');
+    }
+  };
+  walk(coreSrcDir);
+  return hash.digest('hex');
+}
+
+// One decision per package per process, so the hash runs and the warning prints once.
+const useVendoredByPkg = new Map();
+
+// A published install has only vendor-core/. In the monorepo, vendor-core/ is a leftover from a
+// local `npm pack` (it is rebuilt only at prepack), so it is used only while it still matches
+// core/src; otherwise the run says so on stderr and loads core/src, rather than silently running
+// outdated detectors.
+function useVendored(pkgDir) {
+  if (useVendoredByPkg.has(pkgDir)) return useVendoredByPkg.get(pkgDir);
+  const vendorDir = join(pkgDir, 'vendor-core');
+  const srcDir = join(pkgDir, '..', 'core', 'src');
+  let use = existsSync(vendorDir);
+  if (use && existsSync(join(srcDir, 'load-vendored.js'))) {
+    const stampPath = join(vendorDir, SOURCE_HASH_FILE);
+    const stamp = existsSync(stampPath) ? readFileSync(stampPath, 'utf8').trim() : null;
+    if (stamp !== coreSourceHash(srcDir)) {
+      process.stderr.write(
+        `sparkforensics: ${vendorDir} was built from older packages/core sources, so it would run outdated analysis. `
+        + `Using packages/core/src instead; rebuild it with \`node scripts/vendor-core.mjs ${pkgDir}\` or delete it.\n`,
+      );
+      use = false;
+    }
+  }
+  useVendoredByPkg.set(pkgDir, use);
+  return use;
+}
+
 // moduleName is a path relative to core/src without extension, e.g. 'cli/collect-run'. Pass
 // srcExt: 'js' for modules already plain JS in core/src (e.g. 'proxy') rather than TypeScript.
 export function resolveVendored(pkgDir, moduleName, { srcExt = 'ts' } = {}) {
   const vendored = join(pkgDir, 'vendor-core', `${moduleName}.js`);
-  return existsSync(vendored) ? vendored : join(pkgDir, '..', 'core', 'src', `${moduleName}.${srcExt}`);
+  return useVendored(pkgDir) && existsSync(vendored) ? vendored : join(pkgDir, '..', 'core', 'src', `${moduleName}.${srcExt}`);
+}
+
+/** The build id of the core this package runs: the stamped source hash of its vendor-core/ when
+ * that copy is in use, else the hash of core/src computed now. The same `coreSourceHash` the web
+ * build stamps into its bundle, so equal ids mean the same analysis code. 'dev' when neither
+ * exists. */
+export function coreBuildId(pkgDir) {
+  if (useVendored(pkgDir)) {
+    const stampPath = join(pkgDir, 'vendor-core', SOURCE_HASH_FILE);
+    if (existsSync(stampPath)) return readFileSync(stampPath, 'utf8').trim();
+  }
+  const srcDir = join(pkgDir, '..', 'core', 'src');
+  return existsSync(srcDir) ? coreSourceHash(srcDir) : 'dev';
 }
 
 export async function loadVendored(pkgDir, moduleName, opts) {

package/vendor-core/mcp-server-factory.js

@@ -5,6 +5,7 @@ import {
   resolveOrCreateRun, diagnoseRun, getRunSummary, compareRuns, getFindingEvidence, getFindingDocumentation, getReferenceDoc, evaluateBudgetsForRun,
 } from './mcp-tools.js';
 import { listRuns } from './list-runs.js';
+                                                         
 
 const sourceSchema = z.union([
   z.object({ path: z.string() }),
@@ -59,7 +60,9 @@ function toolResult                  (promise            )
   return promise.then((value) => toCallToolResult(JSON.stringify(value), value                           ), toolErrorResult);
 }
 
-export function createMcpServer()            {
+/** `thresholds`: the bin's --thresholds overrides, applied to every analyzing tool for the life
+ * of the server. Omitted, every tool runs the specification defaults. */
+export function createMcpServer({ thresholds }                                      = {})            {
   const server = new McpServer({ name: 'sparkforensics', version: '1.0.0' });
 
   server.registerTool('list_runs', {
@@ -68,7 +71,7 @@ export function createMcpServer()            {
   }, (params) => toolResult(listRuns(params)));
 
   server.registerTool('diagnose_run', {
-    description: 'Diagnose a Spark run: thresholded findings with remediation text, an impact-ranked fix recommendation rollup, and clean-check status.',
+    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, and the checks the log lacked the data to run. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
     inputSchema: {
       ...runRefSchema, redact: z.boolean().optional(),
       include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
@@ -77,14 +80,14 @@ export function createMcpServer()            {
     },
   }, ({ source, runId, redact, include, format, impactBand, type, stageId }) => toolResultWithMarkdown(
     resolveOrCreateRun({ source, runId }).then(({ runId: id }) =>
-      diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId })),
+      diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId, thresholds })),
   ));
 
   server.registerTool('get_run_summary', {
-    description: 'App/stage/job/sql counts and duration for a run, no findings.',
+    description: 'App/stage/job/sql counts, duration, and how the run ended (failed/total jobs and Spark\'s first-line failure reason) for a run, no findings.',
     inputSchema: { ...runRefSchema, redact: z.boolean().optional() },
   }, ({ source, runId, redact }) => toolResult(
-    resolveOrCreateRun({ source, runId }).then(({ runId: id }) => getRunSummary(id, { redact })),
+    resolveOrCreateRun({ source, runId }).then(({ runId: id }) => getRunSummary(id, { redact, thresholds })),
   ));
 
   server.registerTool('compare_runs', {
@@ -96,17 +99,17 @@ export function createMcpServer()            {
       ...formatSchema,
     },
   }, ({ runIdA, sourceA, runIdB, sourceB, redact, format }) => toolResultWithMarkdown(
-    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md' }),
+    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds }),
   ));
 
   server.registerTool('evaluate_budgets', {
-    description: 'Evaluate a run (optionally against a second run for regression budgets) against pass/fail thresholds.',
+    description: 'Evaluate a run against pass/fail thresholds, optionally against a baseline run for regression budgets. With two runs, absolute budgets apply to the candidate (sourceB/runIdB), matching the CLI. A run with no ApplicationEnd always adds an inconclusive run-complete result.',
     inputSchema: {
-      source: sourceSchema.optional().describe('The run to evaluate. Also the regression baseline when sourceB/runIdB is given.'),
+      source: sourceSchema.optional().describe('The run to evaluate. When sourceB/runIdB is given, this is the regression baseline instead, and the absolute budgets apply to sourceB/runIdB.'),
       runId: runRefSchema.runId.describe('Same as `source`, referencing an already-resolved run by id.'),
       maxRuntimeMs: z.number().optional(), maxSpillGb: z.number().optional(), maxSkewRatio: z.number().optional(),
       maxFailedTaskRatePct: z.number().optional(), minEfficiencyPct: z.number().optional(),
-      sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced).'),
+      sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced). When given, the absolute budgets (maxRuntimeMs etc.) are evaluated on this run.'),
       runIdB: secondRunRefSchema.runIdB.describe('Same as `sourceB`, referencing an already-resolved run by id.'),
       maxRegressionPct: z.number().optional(), regressionMetric: z.string().optional(), failOnIntroduced: z.string().optional(),
     },
@@ -118,13 +121,14 @@ export function createMcpServer()            {
     { source, runId },
     { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced },
     (runIdB || sourceB) ? { runId: runIdB, source: sourceB } : undefined,
+    { thresholds },
   )));
 
   server.registerTool('get_finding_evidence', {
     description: 'Raw evidence bundle backing one finding, for drill-down after diagnose_run.',
     inputSchema: { runId: z.string(), findingId: z.string(), redact: z.boolean().optional() },
   }, ({ runId, findingId, redact }) => toolResult(
-    Promise.resolve().then(() => getFindingEvidence(runId, findingId, { redact })),
+    Promise.resolve().then(() => getFindingEvidence(runId, findingId, { redact, thresholds })),
   ));
 
   server.registerTool('get_finding_documentation', {