sparkforensics-mcp 0.3.0 → 0.5.0

package/vendor-core/impact-format.js

@@ -0,0 +1,115 @@
+// Savings figures as every surface prints them: the dashboard's widgets and verdict, and the
+// CLI/MCP evidence report. One formatter set, so units and rounding never drift between paths.
+import { fmtMs, formatRawWaste, formatWallClockRange, readsAsZero } from './format-utils.js';
+                                                                          
+
+/** A finding's one-line savings figure: the wall-clock range for a time-based
+ * finding, the raw resource figure for a `resourceOnly` one, or nothing for a
+ * purely informational estimate or a raw figure that rounds to zero ("0.0
+ * core-h" reads as a measured nothing). */
+export function impactFigure(finding         )                {
+  const estimate = finding.impactEstimate;
+  if (!estimate) return null;
+  if (estimate.wallClock) return formatWallClockRange(estimate.wallClock.low, estimate.wallClock.high);
+  if (estimate.rawWaste) {
+    const text = formatRawWaste(estimate.rawWaste);
+    return readsAsZero(text) ? null : text;
+  }
+  return null;
+}
+
+/** What a raw-waste figure counts, by its unit, as the words that follow it: an idle core
+ * figure is capacity no task ran on, not core time. */
+export function rawWasteMeaning(rawWaste                            )                {
+  switch (rawWaste?.unit) {
+    case 'mbSeconds': return 'of unused executor memory';
+    case 'coreHours':
+    case 'coreMs': return rawWaste.idle ? 'of idle core capacity' : 'of core time';
+    case 'bytes': return 'of extra data written';
+    case 'ms': return 'of task time';
+    default: return null;
+  }
+}
+
+/** What a savings figure counts, as the words that follow it: run time for a
+ * wall-clock claim, or the resource a cost-only (`resourceOnly`) figure
+ * measures. A time figure and a capacity figure look alike ("58.6s",
+ * "0.7 core-h") but only the first shortens the run. Null when the finding
+ * shows no figure. */
+export function savingsMeaning(finding         )                {
+  const estimate = finding.impactEstimate;
+  if (!estimate) return null;
+  if (estimate.wallClock) return 'of run time';
+  return rawWasteMeaning(estimate.rawWaste);
+}
+
+/** A finding's "Potential savings" figure as the widget board shows it: the
+ * wall-clock range, or the raw waste only when there is no wall-clock claim,
+ * never both (they would read as two competing numbers). A zero-value
+ * estimate is suppressed like an informational one: "0s" reads as a measured
+ * figure. `meaning` says what the shown figure counts. */
+export function impactEstimateFigure(estimate                            )                                                  {
+  if (!estimate) return null;
+  const highText = estimate.wallClock && estimate.wallClock.high > 0 ? fmtMs(estimate.wallClock.high) : null;
+  if (highText && !readsAsZero(highText)) {
+    return { text: formatWallClockRange(estimate.wallClock .low, estimate.wallClock .high), meaning: 'of run time' };
+  }
+  const rawWasteText = estimate.rawWaste && estimate.rawWaste.value > 0 ? formatRawWaste(estimate.rawWaste) : null;
+  if (rawWasteText && !readsAsZero(rawWasteText)) return { text: rawWasteText, meaning: rawWasteMeaning(estimate.rawWaste) };
+  return null;
+}
+
+/** Compact single-value form for dense lists (the stage table's finding chips,
+ * the stage dialog): the high-end wall-clock figure, or the raw-waste figure
+ * when there's no wall-clock claim, or `null` for a purely informational or a
+ * zero-value estimate (a `0s` in the spot a real estimate goes would read as a
+ * measured nothing). */
+export function impactEstimateCompact(estimate                            )                {
+  if (!estimate) return null;
+  if (estimate.wallClock) {
+    if (estimate.wallClock.high <= 0) return null;
+    const text = fmtMs(estimate.wallClock.high);
+    return readsAsZero(text) ? null : text;
+  }
+  if (estimate.rawWaste) {
+    if (estimate.rawWaste.value <= 0) return null;
+    const text = formatRawWaste(estimate.rawWaste);
+    return readsAsZero(text) ? null : text;
+  }
+  return null;
+}
+
+/** How a step's savings figure was derived, in one plain sentence for
+ * Advanced view: the estimate method, whether the stage ran alone (a
+ * near-point figure) or shared the cluster (a floor and an optimistic high),
+ * and the raw waste behind it. Null when the finding carries no estimate
+ * model (`estimateMethod: 'none'`), no estimate at all, or a figure that
+ * reads as zero (the step shows no savings then either). Uses the same
+ * formatting and zero rules as the step's own savings figure, which it does
+ * not repeat: the step already shows it. */
+export function estimateProvenance(finding                                 )                {
+  const estimate = finding.impactEstimate;
+  if (!estimate || estimate.estimateMethod === 'none') return null;
+  const method = `${estimate.estimateMethod[0].toUpperCase()}${estimate.estimateMethod.slice(1)}`;
+  const rawWaste = estimate.rawWaste && estimate.rawWaste.value > 0 ? estimate.rawWaste : null;
+  const raw = rawWaste && !readsAsZero(formatRawWaste(rawWaste)) ? formatRawWaste(rawWaste) : null;
+  const wallClock = estimate.wallClock;
+  if (estimate.basis === 'resourceOnly') {
+    return raw ? `${method}; ${raw} wasted, which may not shorten the run.` : null;
+  }
+  if (!wallClock || wallClock.high <= 0) return null;
+  const highText = formatWallClockRange(wallClock.high, wallClock.high);
+  if (readsAsZero(highText)) return null;
+  let rawNote = '';
+  if (raw && rawWaste .unit !== 'ms') rawNote = ` Resource waste measured: ${raw}.`;
+  else if (raw && rawWaste .value > wallClock.high && raw !== highText) rawNote = ` Raw waste before the floor clipped it: ${raw}.`;
+  if (estimate.basis === 'serial') {
+    return `${method}; the stage ran alone, so this is close to a point estimate.${rawNote}`;
+  }
+  if (estimate.basis === 'contended') {
+    const lowText = formatWallClockRange(wallClock.low, wallClock.low);
+    const spread = lowText === highText ? 'its floor and high agree' : `${lowText} is the floor, ${highText} if the fix fully lands`;
+    return `${method}; the stage shared the cluster: ${spread}.${rawNote}`;
+  }
+  return null;
+}

package/vendor-core/impact-model.js

@@ -0,0 +1,197 @@
+// The waste models every detector entry's estimate() builds its ImpactEstimate from: the assumed
+// throughputs, the per-stage measurements behind them and the occupancy clip wrappers. Each
+// finding type's own composition of these lives on its DETECTORS entry, next to its detect().
+                                                                                                                     
+import { isPythonStage } from './python-stage.js';
+import { nsToMs } from './format-utils.js';
+import {
+  estimateSingleStage, estimateMultiStage,
+                                                                                
+} from './occupancy.js';
+
+/** What every estimate() is handed: built once per analyze(), and the same object detect() gates
+ * its runtime floors against (DetectorCtx.impact), so a floor and the savings displayed for it
+ * read one occupancy sweep. */
+                              
+                             
+                                             
+                                                                                 
+                     
+                                                                                                  
+                                  
+ 
+
+// Assumed shuffle-network throughput per executor link, ~1 Gbps. Starting assumption, unvalidated.
+export const SHUFFLE_THROUGHPUT_BPS = 125_000_000;
+// Assumed disk I/O throughput per executor for spilled data, ~200 MB/s (conservative HDD/SSD blend).
+export const SPILL_IO_THROUGHPUT_BPS = 200_000_000;
+
+// A stage's own per-task overhead: task wall time (launch to finish, summed per executor by
+// finalizeStage) minus executorRunTime, i.e. deserialization, result serialization and the
+// launch round trip that coalescing tasks removes. `concurrency` is the stage's achieved task
+// concurrency (task time / stage duration). Null when the stage has no such data or no overhead.
+export function measuredTaskOverhead(stage       )                                                    {
+  const executorStats = Array.isArray(stage.executorStats)
+    ? (stage.executorStats                                ) : [];
+  const taskTimeMs = executorStats.reduce((sum, e) => sum + (e.totalDuration ?? 0), 0);
+  const taskCount = stage.taskCount ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  const overheadMs = taskTimeMs - (stage.executorRunTime ?? 0);
+  if (taskTimeMs <= 0 || taskCount <= 0 || durationMs <= 0 || overheadMs <= 0) return null;
+  return { perTaskMs: overheadMs / taskCount, concurrency: taskTimeMs / durationMs };
+}
+
+// Both constants above are single-device figures (one NIC, one local disk). A stage's shuffle
+// reads and spills are spread over every executor that ran its tasks, each moving its own share
+// in parallel, so the stage's aggregate bandwidth scales with that executor count. Dividing a
+// stage-wide byte total by one device's bandwidth modeled the whole cluster as one link: on 14
+// real logs that claimed up to 1,870s of shuffle time on stages whose tasks measured ~0s of
+// shuffle fetch wait (222 of 284 shuffle findings). No executor data falls back to one device.
+export function stageIoParallelism(stage       )         {
+  const executors = Array.isArray(stage.executorStats) ? stage.executorStats.length : 0;
+  return Math.max(1, executors);
+}
+
+// Wall-clock the stage's tasks spent blocked fetching shuffle blocks: fetchWaitTime is a
+// cross-task sum like executorRunTime, so dividing by the stage's average concurrency converts it
+// (the gc estimate's conversion). Null when the stage has no run time or duration to convert with.
+// On 14 real logs the link model claimed 2780s over 284 shuffle findings, 256s once capped at
+// this, with about zero fetch wait on 5 of the 10 non-info ones: reads that overlapped compute
+// stalled nothing.
+export function fetchWaitWallClockMs(stage       )                {
+  const fetchWaitMs = stage.fetchWaitTime;
+  const runTimeMs = stage.executorRunTime ?? 0;
+  const durationMs = (stage.completedAt ?? 0) - (stage.submittedAt ?? 0);
+  if (typeof fetchWaitMs !== 'number' || runTimeMs <= 0 || durationMs <= 0) return null;
+  return fetchWaitMs / (runTimeMs / durationMs);
+}
+
+// Wall-clock a stage's wasted (retried) attempts cost it. Each one delayed only its own task, and
+// attempts of different tasks ran side by side: one lost executor fails every task it was running
+// at once (4 wasted attempts of 36.6s each, all first attempts, on a real stage that ran 41 tasks
+// at once, claimed as 146.6s). So the stage lost at most its longest retry chain, the most attempts
+// one task wasted (its highest attempt number + 1) times the mean wasted attempt, or their summed
+// time spread over its slots, whichever is larger. Without a sample of every wasted attempt
+// (retryTaskSamples is capped), the chain isn't known: the summed time.
+export function retryWallClockMs(stage       )         {
+  const totalMs = (stage.retryWasteMs                      ) ?? 0;
+  const attempts = (stage.wastedAttempts                      ) ?? 0;
+  const samples = Array.isArray(stage.retryTaskSamples)
+    ? (stage.retryTaskSamples                                ) : [];
+  if (totalMs <= 0 || attempts <= 0 || samples.length < attempts) return totalMs;
+  const chain = samples.reduce((longest, s) => Math.max(longest, (s.attemptNumber ?? 0) + 1), 1);
+  const slots = Math.max(1, stage.peakConcurrentTasks ?? 1);
+  return Math.min(totalMs, Math.max((chain * totalMs) / attempts, totalMs / slots));
+}
+// Spark's classic recommended shuffle partition size.
+export const IDEAL_BYTES_PER_PARTITION_TASK = 128 * 1024 * 1024;
+// Assumed per-task scheduling/launch overhead: the fallback when a stage lacks the per-executor
+// task-time sums measuredTaskOverhead needs.
+export const TASK_SCHEDULING_OVERHEAD_MS = 50;
+// Assumed per-file open latency (small-file overhead).
+export const FILE_OPEN_OVERHEAD_MS = 10;
+// Assumed broadcast-transfer bandwidth, shared with overBroadcast/underBroadcast.
+export const BROADCAST_BANDWIDTH_BPS = 125_000_000;
+// Assumed per-non-local-task network-fetch penalty, reported as extra core-time.
+export const NETWORK_FETCH_PENALTY_MS = 20;
+// Assumed executor JVM+container startup overhead.
+export const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
+// Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
+export const RE_READ_THROUGHPUT_BPS = 125_000_000;
+
+// Below this share of executorRunTime spent on CPU, a stage's tasks were idle, waiting on something
+// outside Spark: on 14 real logs (2026-09-23) every non-Python stage under 1% was a JDBC read, a
+// file listing or a Delta log read, while file writes, which more partitions do parallelize,
+// start at 2%.
+const IDLE_CPU_SHARE_MAX = 0.01;
+
+// True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
+// the share can't be trusted: no CPU time recorded (older Spark logs omit the metric), or Python
+// code run through a Python worker (isPythonStage: a PythonRDD stage or a Python UDF operator in
+// its SQL plan), whose worker-process CPU executorCpuTime (the JVM task thread's) never counts
+// (such stages read 0.1% on the same logs while computing).
+export function tasksMostlyIdle(stage       , sql                            = new Map())          {
+  const runMs = stage.executorRunTime ?? 0;
+  const cpuMs = nsToMs(stage.executorCpuTime ?? 0);
+  if (runMs <= 0 || cpuMs <= 0) return false;
+  if (isPythonStage(stage, sql)) return false;
+  return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
+}
+
+// skew, straggler and stageSlowness claim time off the stage's longest task itself, so the
+// occupancy clip must not floor them at that same task (see estimateSingleStage).
+export const TAIL_CLAIM                             = { shortensLongestTask: true };
+
+// No quantifiable magnitude -> 'informational'; a rawWaste figure with no stage window ->
+// 'resourceOnly'. Never a fake {low:0, high:0}: wallClock is null in both cases.
+export function costOnly(estimateMethod                      , rawWaste                 )                 {
+  return rawWaste
+    ? { basis: 'resourceOnly', wallClock: null, estimateMethod, rawWaste }
+    : { basis: 'informational', wallClock: null, estimateMethod };
+}
+
+/** The estimate() of an entry whose findings have no waste model at all. */
+export function noWasteModel()                 {
+  return costOnly('none');
+}
+
+export function singleStageImpact(
+  wasteMs        ,
+  stageId        ,
+  ctx             ,
+  estimateMethod                      ,
+  rawWaste                 ,
+  opts                             ,
+)                 {
+  const est = estimateSingleStage(wasteMs, stageId, ctx.stages                                          , ctx.occupancy, opts);
+  if (est) return { basis: est.basis, wallClock: est.wallClock, estimateMethod, rawWaste };
+  return costOnly(estimateMethod, rawWaste); // stage excluded from the sweep (duration <= 0)
+}
+
+/** estimateMultiStage over the finding's own stages, or null when every one was excluded from
+ * the sweep. */
+export function multiStageImpact(
+  stageIds          ,
+  wasteMsByStage                     ,
+  ctx             ,
+  estimateMethod                      ,
+  rawWaste                 ,
+)                        {
+  const est = estimateMultiStage(stageIds, wasteMsByStage, ctx.stages                                          , ctx.occupancy);
+  return est ? { basis: est.basis, wallClock: est.wallClock, estimateMethod, rawWaste } : null;
+}
+
+export function stageMappableWasteOrCostOnly(
+  wasteMs        ,
+  stageIds                      ,
+  ctx             ,
+)                 {
+  const rawWaste                             = wasteMs > 0 ? { value: wasteMs, unit: 'ms' } : undefined;
+  if (!stageIds || stageIds.length === 0) {
+    return costOnly('modeled', rawWaste);
+  }
+  // One waste event spread over a span of stages, not N independent wastes: apportion evenly so
+  // estimateMultiStage's union cap doesn't absorb the same amount claimed once per stage.
+  const perStageWasteMs = wasteMs / stageIds.length;
+  const wasteMsByStage = new Map(stageIds.map((id) => [id, perStageWasteMs]));
+  // null: every stage excluded from the sweep
+  return multiStageImpact(stageIds, wasteMsByStage, ctx, 'modeled', rawWaste) ?? costOnly('modeled', rawWaste);
+}
+
+// Finding types whose raw figure is busy core time read straight from the log: gc's jvmGCTime
+// (coreMs) and the discarded speculative or retried attempts' run time ('ms' cross-task sums).
+// Any other core figure is idle capacity, or modeled on an assumed constant (coreLocality's
+// per-task fetch penalty, autoscalingChurn's executor-hours, jobFailureRate's job-hours).
+const MEASURED_CORE_TIME_FIGURE                      = new Set(['gc', 'retryWaste', 'speculationWaste']);
+
+/** The busy core time a finding's fix removes, in core-milliseconds, or null when the detector
+ * measures none. Only a measured figure counts: one its estimate() already set (skew and
+ * straggler's removed task time), or the raw figure of a MEASURED_CORE_TIME_FIGURE type. A
+ * wall-clock claim, an idle capacity figure and a modeled figure are never converted. It never
+ * reads executorCpuTime, which leaves out Python worker CPU. */
+export function coreTimeFor(finding         , estimate                )                                       {
+  if (estimate.coreTimeMs !== undefined) return estimate.coreTimeMs;
+  const raw = estimate.rawWaste;
+  if (!raw || !MEASURED_CORE_TIME_FIGURE.has(finding.type)) return null;
+  return { low: raw.value, high: raw.value };
+}

package/vendor-core/ingest.js

@@ -13,6 +13,8 @@
                                   
                                             
                                                    
+                                                    
+                                                   
                                    
                                     
  
@@ -37,9 +39,11 @@ export function routeMessage(
     case 'job':      handlers.onJob?.(data.data); break;
     case 'runAggregates': handlers.onRunAggregates?.(data.data); break;
     case 'stageExecutorMetrics': handlers.onStageExecutorMetrics?.(data.data); break;
+    case 'stageSpeculationWaste': handlers.onStageSpeculationWaste?.(data.data); break;
+    case 'stageLateAttemptWork': handlers.onStageLateAttemptWork?.(data.data); break;
     case 'done': {
-      const { skippedLines } = data;
-      handlers.onDone?.({ skippedLines });
+      const { skippedLines, unreadableSqlExecutions } = data;
+      handlers.onDone?.(unreadableSqlExecutions ? { skippedLines, unreadableSqlExecutions } : { skippedLines });
       break;
     }
     case 'error':    handlers.onError?.(data); break;

package/vendor-core/intervals.js

@@ -0,0 +1,13 @@
+// Interval union shared by the wall-clock breakdown and the savings rollups.
+export function mergeIntervals(intervals                    )                     {
+  if (intervals.length === 0) return [];
+  const sorted = [...intervals].sort((a, b) => a[0] - b[0]);
+  const out                     = [[sorted[0][0], sorted[0][1]]];
+  for (let i = 1; i < sorted.length; i++) {
+    const last = out[out.length - 1];
+    const cur = sorted[i];
+    if (cur[0] <= last[1]) last[1] = Math.max(last[1], cur[1]);
+    else out.push([cur[0], cur[1]]);
+  }
+  return out;
+}

package/vendor-core/list-runs.js

@@ -5,6 +5,7 @@ import { reassembleRollingEntries } from './parser-worker.js';
 import { peekLogHeader } from './log-header-peek.js';
 import { mcpError } from './mcp-error.js';
 import { normalizeBaseUrl } from './shs-request.js';
+import { isConnectionFailure } from './proxy.js';
 import { DEFAULT_IDLE_TIMEOUT_MS, DEFAULT_MAX_ARCHIVE_BYTES } from './shs-load.js';
 
                                
@@ -82,9 +83,8 @@ export function applyFiltersAndCap(entries                , filters
   return filters.redact ? { ...capped, runs: redactRunListEntries(capped.runs) } : capped;
 }
 
-// Local counterpart to redact.ts's redactAppIdentity, sized for a listing of many distinct apps
-// rather than the single app that tool works over: redactAppIdentity has no way to keep two
-// entries sharing an appId in sync, so this builds its own stable per-distinct-appId pseudonym map
+// Listing-wide app identity redaction: redact.ts's redactors each work over one run, with no way
+// to keep two listing entries sharing an appId in sync, so this builds its own stable per-distinct-appId pseudonym map
 // (numeric-aware sort, same scheme as redact.ts's buildMap) across the whole result set, two
 // attempts of the same app redact to the same identity, and applies it to every identity-bearing
 // field: appId, name (a listing's human-readable name is as identifying as the id itself), and the
@@ -206,10 +206,12 @@ export async function listRunsShs(
   let res          ;
   try {
     // An unresponsive SHS would otherwise hang the tool call forever. A timed-out signal rejects
-    // the fetch with an AbortError, which this same catch turns into access-or-upstream-failure.
+    // the fetch with a TimeoutError, which this same catch turns into upstream-unreachable, the
+    // code a refused connection gets too.
     res = await fetchImpl(url.toString(), { signal: AbortSignal.timeout(DEFAULT_IDLE_TIMEOUT_MS) });
   } catch (e) {
-    throw mcpError('access-or-upstream-failure', `Could not reach ${normalized}: ${e instanceof Error ? e.message : String(e)}`);
+    const code = isConnectionFailure(e) ? 'upstream-unreachable' : 'access-or-upstream-failure';
+    throw mcpError(code, `Could not reach ${normalized}: ${e instanceof Error ? e.message : String(e)}`);
   }
   if (!res.ok) {
     throw mcpError('access-or-upstream-failure', `SHS applications list request failed with status ${res.status}.`);

package/vendor-core/load-vendored.js

@@ -1,16 +1,81 @@
 // Plain JS: scripts/vendor-core.mjs copies it byte-for-byte into vendor-core/, so this file exists
 // at both vendor-core/load-vendored.js (published) and core/src/load-vendored.js (dev). Each
-// package's bin bootstraps by locating this file first, then uses the exports below for every other
-// core module, so the resolution logic lives in one place instead of per entry point.
-import { existsSync } from 'node:fs';
-import { join } from 'node:path';
+// package's bin bootstraps by locating this file first (the core/src copy when it exists, so the
+// staleness check below is always current code), then uses the exports below for every other core
+// module, so the resolution logic lives in one place instead of per entry point.
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
 import { pathToFileURL } from 'node:url';
 
+// Written into vendor-core/ by scripts/vendor-core.mjs: the coreSourceHash of the core/src it was
+// built from, so a monorepo run can tell a leftover vendored copy from a current one.
+export const SOURCE_HASH_FILE = 'core-source-hash.txt';
+
+// docs-content/ is generated from a pinned upstream commit, not analysis code.
+const UNHASHED_TOP_LEVEL = new Set(['docs-content']);
+
+/** A content hash of every analysis source file under core/src (paths and bytes, sorted). */
+export function coreSourceHash(coreSrcDir) {
+  const hash = createHash('sha256');
+  const walk = (dir) => {
+    const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+      if (dir === coreSrcDir && UNHASHED_TOP_LEVEL.has(entry.name)) continue;
+      const path = join(dir, entry.name);
+      if (entry.isDirectory()) walk(path);
+      else if (entry.isFile()) hash.update(`${relative(coreSrcDir, path)}\0`).update(readFileSync(path)).update('\0');
+    }
+  };
+  walk(coreSrcDir);
+  return hash.digest('hex');
+}
+
+// One decision per package per process, so the hash runs and the warning prints once.
+const useVendoredByPkg = new Map();
+
+// A published install has only vendor-core/. In the monorepo, vendor-core/ is a leftover from a
+// local `npm pack` (it is rebuilt only at prepack), so it is used only while it still matches
+// core/src; otherwise the run says so on stderr and loads core/src, rather than silently running
+// outdated detectors.
+function useVendored(pkgDir) {
+  if (useVendoredByPkg.has(pkgDir)) return useVendoredByPkg.get(pkgDir);
+  const vendorDir = join(pkgDir, 'vendor-core');
+  const srcDir = join(pkgDir, '..', 'core', 'src');
+  let use = existsSync(vendorDir);
+  if (use && existsSync(join(srcDir, 'load-vendored.js'))) {
+    const stampPath = join(vendorDir, SOURCE_HASH_FILE);
+    const stamp = existsSync(stampPath) ? readFileSync(stampPath, 'utf8').trim() : null;
+    if (stamp !== coreSourceHash(srcDir)) {
+      process.stderr.write(
+        `sparkforensics: ${vendorDir} was built from older packages/core sources, so it would run outdated analysis. `
+        + `Using packages/core/src instead; rebuild it with \`node scripts/vendor-core.mjs ${pkgDir}\` or delete it.\n`,
+      );
+      use = false;
+    }
+  }
+  useVendoredByPkg.set(pkgDir, use);
+  return use;
+}
+
 // moduleName is a path relative to core/src without extension, e.g. 'cli/collect-run'. Pass
 // srcExt: 'js' for modules already plain JS in core/src (e.g. 'proxy') rather than TypeScript.
 export function resolveVendored(pkgDir, moduleName, { srcExt = 'ts' } = {}) {
   const vendored = join(pkgDir, 'vendor-core', `${moduleName}.js`);
-  return existsSync(vendored) ? vendored : join(pkgDir, '..', 'core', 'src', `${moduleName}.${srcExt}`);
+  return useVendored(pkgDir) && existsSync(vendored) ? vendored : join(pkgDir, '..', 'core', 'src', `${moduleName}.${srcExt}`);
+}
+
+/** The build id of the core this package runs: the stamped source hash of its vendor-core/ when
+ * that copy is in use, else the hash of core/src computed now. The same `coreSourceHash` the web
+ * build stamps into its bundle, so equal ids mean the same analysis code. 'dev' when neither
+ * exists. */
+export function coreBuildId(pkgDir) {
+  if (useVendored(pkgDir)) {
+    const stampPath = join(pkgDir, 'vendor-core', SOURCE_HASH_FILE);
+    if (existsSync(stampPath)) return readFileSync(stampPath, 'utf8').trim();
+  }
+  const srcDir = join(pkgDir, '..', 'core', 'src');
+  return existsSync(srcDir) ? coreSourceHash(srcDir) : 'dev';
 }
 
 export async function loadVendored(pkgDir, moduleName, opts) {

package/vendor-core/mcp-server-factory.js

@@ -5,6 +5,7 @@ import {
   resolveOrCreateRun, diagnoseRun, getRunSummary, compareRuns, getFindingEvidence, getFindingDocumentation, getReferenceDoc, evaluateBudgetsForRun,
 } from './mcp-tools.js';
 import { listRuns } from './list-runs.js';
+                                                         
 
 const sourceSchema = z.union([
   z.object({ path: z.string() }),
@@ -59,7 +60,9 @@ function toolResult                  (promise            )
   return promise.then((value) => toCallToolResult(JSON.stringify(value), value                           ), toolErrorResult);
 }
 
-export function createMcpServer()            {
+/** `thresholds`: the bin's --thresholds overrides, applied to every analyzing tool for the life
+ * of the server. Omitted, every tool runs the specification defaults. */
+export function createMcpServer({ thresholds }                                      = {})            {
   const server = new McpServer({ name: 'sparkforensics', version: '1.0.0' });
 
   server.registerTool('list_runs', {
@@ -68,7 +71,7 @@ export function createMcpServer()            {
   }, (params) => toolResult(listRuns(params)));
 
   server.registerTool('diagnose_run', {
-    description: 'Diagnose a Spark run: thresholded findings with remediation text, an impact-ranked fix recommendation rollup, and clean-check status.',
+    description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings), thresholded findings with remediation text, an impact-ranked fix recommendation rollup, clean-check status, and the checks the log lacked the data to run. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated.',
     inputSchema: {
       ...runRefSchema, redact: z.boolean().optional(),
       include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
@@ -77,14 +80,14 @@ export function createMcpServer()            {
     },
   }, ({ source, runId, redact, include, format, impactBand, type, stageId }) => toolResultWithMarkdown(
     resolveOrCreateRun({ source, runId }).then(({ runId: id }) =>
-      diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId })),
+      diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId, thresholds })),
   ));
 
   server.registerTool('get_run_summary', {
-    description: 'App/stage/job/sql counts and duration for a run, no findings.',
+    description: 'App/stage/job/sql counts, duration, and how the run ended (failed/total jobs and Spark\'s first-line failure reason) for a run, no findings.',
     inputSchema: { ...runRefSchema, redact: z.boolean().optional() },
   }, ({ source, runId, redact }) => toolResult(
-    resolveOrCreateRun({ source, runId }).then(({ runId: id }) => getRunSummary(id, { redact })),
+    resolveOrCreateRun({ source, runId }).then(({ runId: id }) => getRunSummary(id, { redact, thresholds })),
   ));
 
   server.registerTool('compare_runs', {
@@ -96,17 +99,17 @@ export function createMcpServer()            {
       ...formatSchema,
     },
   }, ({ runIdA, sourceA, runIdB, sourceB, redact, format }) => toolResultWithMarkdown(
-    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md' }),
+    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds }),
   ));
 
   server.registerTool('evaluate_budgets', {
-    description: 'Evaluate a run (optionally against a second run for regression budgets) against pass/fail thresholds.',
+    description: 'Evaluate a run against pass/fail thresholds, optionally against a baseline run for regression budgets. With two runs, absolute budgets apply to the candidate (sourceB/runIdB), matching the CLI. A run with no ApplicationEnd always adds an inconclusive run-complete result.',
     inputSchema: {
-      source: sourceSchema.optional().describe('The run to evaluate. Also the regression baseline when sourceB/runIdB is given.'),
+      source: sourceSchema.optional().describe('The run to evaluate. When sourceB/runIdB is given, this is the regression baseline instead, and the absolute budgets apply to sourceB/runIdB.'),
       runId: runRefSchema.runId.describe('Same as `source`, referencing an already-resolved run by id.'),
       maxRuntimeMs: z.number().optional(), maxSpillGb: z.number().optional(), maxSkewRatio: z.number().optional(),
       maxFailedTaskRatePct: z.number().optional(), minEfficiencyPct: z.number().optional(),
-      sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced).'),
+      sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced). When given, the absolute budgets (maxRuntimeMs etc.) are evaluated on this run.'),
       runIdB: secondRunRefSchema.runIdB.describe('Same as `sourceB`, referencing an already-resolved run by id.'),
       maxRegressionPct: z.number().optional(), regressionMetric: z.string().optional(), failOnIntroduced: z.string().optional(),
     },
@@ -118,13 +121,14 @@ export function createMcpServer()            {
     { source, runId },
     { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced },
     (runIdB || sourceB) ? { runId: runIdB, source: sourceB } : undefined,
+    { thresholds },
   )));
 
   server.registerTool('get_finding_evidence', {
     description: 'Raw evidence bundle backing one finding, for drill-down after diagnose_run.',
     inputSchema: { runId: z.string(), findingId: z.string(), redact: z.boolean().optional() },
   }, ({ runId, findingId, redact }) => toolResult(
-    Promise.resolve().then(() => getFindingEvidence(runId, findingId, { redact })),
+    Promise.resolve().then(() => getFindingEvidence(runId, findingId, { redact, thresholds })),
   ));
 
   server.registerTool('get_finding_documentation', {