sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/run-shape.js

@@ -0,0 +1,40 @@
+// The run's shape as the dashboard's Scorecard, ETL Phase Attribution and Core Usage by Locality
+// cards show it, for the CLI/MCP paths: every figure comes from the same core function those cards
+// read, and null means the card would show "Not measured" or "Unavailable".
+import { hasFinishedStage } from './check-coverage.js';
+import { buildLocalityChart } from './core-usage-locality.js';
+import { attributeEtlPhases } from './etl-phases.js';
+import { getScorecardEstimates, hasCompleteApplicationInterval } from './scorecard-estimates.js';
+import { computeWallClock } from './wall-clock.js';
+                                           
+
+                           
+                                                                                      
+                             
+                                                                                              
+                                                      
+                               
+                                                                                               
+                                                                                          
+                                   
+                                                                                               
+                                                
+                                                                           
+                                                                                           
+                       
+                               
+ 
+
+export function computeRunShape(appModel          )           {
+  const timed = hasCompleteApplicationInterval(appModel.app);
+  const estimates = getScorecardEstimates(appModel);
+  const phases = attributeEtlPhases(appModel.stages);
+  const chart = buildLocalityChart([...appModel.stages.values()], appModel.app);
+  return {
+    wallClockMs: timed ? computeWallClock(appModel.app, appModel.stages).total : null,
+    efficiencyPct: hasFinishedStage(appModel.stages) ? estimates.efficiency.value : null,
+    unusedCoreTimePct: estimates.wastage.value,
+    etlPhasesMs: phases.extract + phases.transform + phases.load > 0 ? phases : null,
+    peakBusyCores: chart.hasActivity ? chart.peakCores : null,
+  };
+}

package/vendor-core/run-verdict.js

@@ -0,0 +1,353 @@
+// The run verdict: where to start, a short summary, and the top places to look, ranked by
+// potential savings (failures first on a failed run). Shared by the dashboard's verdict card and
+// the CLI/MCP evidence report, so both paths name the same first step in the same words.
+import { ENTRY_BY_TYPE } from './detectors.js';
+import { hasFinishedStage, isCleanRun } from './check-coverage.js';
+import { findingActionLabel } from './finding-action-label.js';
+import { singleStageId } from './finding-filter-predicate.js';
+import { recommendationText } from './finding-names.js';
+import { formatDuration, IMPACT_BAND_ORDER } from './format-utils.js';
+import { impactFigure, savingsMeaning } from './impact-format.js';
+import { isEligible } from './recommendation-rollup.js';
+import { FAILURE_TYPES, quotesReasonOf, summarizeRunOutcome,                 } from './run-outcome.js';
+import { getScorecardEstimates, hasCompleteApplicationInterval } from './scorecard-estimates.js';
+import { computeWallClock } from './wall-clock.js';
+                                                    
+
+/** How many next steps the verdict lists before pointing at the full list. */
+export const NEXT_STEP_LIMIT = 3;
+
+/** Finding types whose widget sits in the board's reference region (listed after every action
+ * widget). The dashboard's registry must agree: tests/view/detector-registry.test.tsx checks it. */
+export const REFERENCE_DISPLAY_TYPES                      = new Set([
+  'memoryUtilization', 'utilization', 'coreLocality', 'cacheUtilization',
+]);
+
+/** Every emitted finding type in the board's widget display order: action region before
+ * reference region, then ascending order of the type's `ENTRY_BY_TYPE` entry.
+ * The last tiebreak of the verdict ranking, so the CLI orders ties exactly as the dashboard. */
+export const FINDING_DISPLAY_ORDER                    = (() => {
+  const region = (type        ) => (REFERENCE_DISPLAY_TYPES.has(type) ? 1 : 0);
+  return [...ENTRY_BY_TYPE.entries()]
+    .sort(([a, entryA], [b, entryB]) => region(a) - region(b) || entryA.order - entryB.order)
+    .map(([type]) => type);
+})();
+
+const DISPLAY_INDEX = new Map(FINDING_DISPLAY_ORDER.map((type, index) => [type, index]));
+
+// The high end of the finding's own occupancy-clipped wall-clock estimate, the figure the
+// "Potential savings" line leads with. `null` with no quantified time claim
+// (resourceOnly/informational basis): such a finding can never win on its own numbers.
+function potentialSavingsMs(finding         )                {
+  return finding.impactEstimate?.wallClock?.high ?? null;
+}
+
+/** True for a finding the verdict can route to: a known display type with a recommendation. */
+export function isRankable(finding         )          {
+  const recommendation = typeof finding.recommendation === 'string' ? finding.recommendation.trim() : '';
+  return DISPLAY_INDEX.has(finding.type) && recommendation.length > 0;
+}
+
+/** Every rankable finding, best first. Ranked by potential savings: a quantified estimate always
+ * outranks an unquantified one; ties (including "neither has one") fall back to impact band,
+ * then widget display order, then input order, so an unquantified warning still leads an info. */
+export function rankBySavings(findings           )            {
+  const candidates = findings
+    .map((finding, index) => ({ finding, index }))
+    .filter(({ finding }) => isRankable(finding));
+  candidates.sort((left, right) => {
+    const leftSavings = potentialSavingsMs(left.finding);
+    const rightSavings = potentialSavingsMs(right.finding);
+    if (leftSavings !== null && rightSavings !== null && leftSavings !== rightSavings) return rightSavings - leftSavings;
+    if ((leftSavings !== null) !== (rightSavings !== null)) return leftSavings !== null ? -1 : 1;
+    return (
+      (IMPACT_BAND_ORDER[left.finding.impactBand] ?? 9) - (IMPACT_BAND_ORDER[right.finding.impactBand] ?? 9)
+      || (DISPLAY_INDEX.get(left.finding.type) ?? Number.MAX_SAFE_INTEGER) - (DISPLAY_INDEX.get(right.finding.type) ?? Number.MAX_SAFE_INTEGER)
+      || left.index - right.index
+    );
+  });
+  return candidates.map(({ finding }) => finding);
+}
+
+/** One place worth a look: the highest-ranked finding at a location, plus the
+ * other finding types flagged at that same location. Findings that share a
+ * stage usually share one root cause, so they read as one step, not several. */
+                           
+              
+                
+                                                                                  
+                     
+                                                              
+                         
+ 
+
+/** A finding's location identity for grouping. Per-stage findings (and
+ * sql-scope findings that touch exactly one stage) group by that stage; a
+ * multi-stage or app-level finding stands alone by its own type (and variant,
+ * for app-level ones), since two different app-level problems are not the
+ * same place. */
+export function locationKey(finding         )                                          {
+  const stageId = singleStageId(finding);
+  if (stageId != null) return { key: `stage:${stageId}`, stageId };
+  if ('stageIds' in finding && finding.stageIds.length > 1) {
+    return { key: `stages:${finding.type}:${[...finding.stageIds].sort((a, b) => a - b).join(',')}`, stageId: null };
+  }
+  const variant = 'variant' in finding ? finding.variant : undefined;
+  return { key: variant ? `app:${finding.type}:${variant}` : `app:${finding.type}`, stageId: null };
+}
+
+/** 0 for a failure at a stage of a failed job (what stopped the job), 1 for
+ * any other failure finding, 2 for everything else. */
+function failureRank(finding         , failedJobStageIds                     )         {
+  if (!FAILURE_TYPES.has(finding.type)) return 2;
+  return typeof finding.stageId === 'number' && failedJobStageIds.has(finding.stageId) ? 0 : 1;
+}
+
+/** Groups every rankable finding by location, ordered by the location's
+ * best-ranked finding, so step 1 is always the run's single biggest win. With
+ * `failedJobStageIds` (a run whose jobs failed), failure findings rank ahead
+ * of every savings figure and lead their location, those at a failed job's
+ * stage first: a speed-up is moot until the job finishes. */
+export function buildNextSteps(findings           , { failedJobStageIds }                                              = {})             {
+  const ranked = rankBySavings(findings);
+  // Array.prototype.sort is stable, so savings order holds within each rank.
+  const ordered = failedJobStageIds
+    ? [...ranked].sort((a, b) => failureRank(a, failedJobStageIds) - failureRank(b, failedJobStageIds))
+    : ranked;
+  const steps = new Map                  ();
+  for (const finding of ordered) {
+    const { key, stageId } = locationKey(finding);
+    const existing = steps.get(key);
+    if (!existing) {
+      steps.set(key, { key, lead: finding, related: [], stageId });
+      continue;
+    }
+    const seenTypes = new Set([existing.lead.type, ...existing.related.map((f) => f.type)]);
+    if (!seenTypes.has(finding.type)) existing.related.push(finding);
+  }
+  return [...steps.values()];
+}
+
+/** True for findings about executor capacity sitting idle. memoryUtilization
+ * also reports heap pressure and over-provisioning, which are not idle
+ * capacity, so only its idleCores variant counts. */
+function isIdleCapacityFinding(finding         )          {
+  return finding.type === 'utilization' || (finding.type === 'memoryUtilization' && finding.variant === 'idleCores');
+}
+
+/** Idle share at which the verdict notes, in its summary, that the cluster
+ * may be larger than the job needs. It never reorders the steps. */
+export const IDLE_NOTABLE_PCT = 40;
+
+export function isIdleCapacityStep(step          )          {
+  return isIdleCapacityFinding(step.lead);
+}
+
+/** The idle share an idle-capacity finding itself reports: idleCores carries
+ * the idle rate, utilization the busy rate. Null for any other finding. */
+function reportedIdlePct(finding         )                {
+  if (typeof finding.value !== 'number') return null;
+  if (finding.type === 'memoryUtilization' && finding.variant === 'idleCores') return finding.value;
+  if (finding.type === 'utilization') return 100 - finding.value;
+  return null;
+}
+
+/** The run's idle share as the verdict states it: the figure the top-ranked
+ * idle-capacity step reports, so the verdict never disagrees with that step,
+ * or `fallbackPct` (the Scorecard's Unused core time) when no step reports one. */
+export function verdictIdlePct(steps            , fallbackPct               )                {
+  const idleStep = steps.find(isIdleCapacityStep);
+  return (idleStep ? reportedIdlePct(idleStep.lead) : null) ?? fallbackPct;
+}
+
+function plural(count        , noun        )         {
+  return `${count} ${noun}${count === 1 ? '' : 's'}`;
+}
+
+/** The run facts the verdict's wording depends on, read once. */
+                           
+                                                                               
+                       
+                                                                         
+                         
+                                                                             
+                      
+                                                                          
+                                                                      
+                 
+                      
+                                                                        
+                            
+                            
+ 
+
+function isFailedRun(facts          )          {
+  return facts.outcome.failedJobs > 0;
+}
+
+/** The failed-run title: the one thing a newcomer must know before any
+ * tuning advice is that the job did not finish. */
+function failedTitle({ failedJobs, totalJobs }            )         {
+  if (failedJobs < totalJobs) return `${failedJobs} of ${totalJobs} jobs failed in this run`;
+  return totalJobs === 1 ? 'This run failed: its job did not finish' : `This run failed: all ${totalJobs} jobs did not finish`;
+}
+
+/** An action label as it reads after "Start here:": only its first letter
+ * drops to lower case, so a name inside it ("Switch to Kryo") keeps its
+ * capital, and a leading acronym ("GC", "OOM") is left alone. */
+function lowerFirst(label        )         {
+  if (/^[A-Z]{2}/.test(label)) return label;
+  return label.charAt(0).toLowerCase() + label.slice(1);
+}
+
+export function verdictTitle(eligible           , steps            , facts          )         {
+  if (isFailedRun(facts)) return failedTitle(facts.outcome);
+  if (eligible.length === 0 && facts.incomplete) return 'This log looks incomplete, so results cover only part of the run';
+  if (eligible.length === 0 && facts.noFinishedStages) return 'This log has no finished stages to check';
+  if (eligible.length === 0 && !facts.clean) return 'Nothing to fix, but some checks could not run on this log';
+  if (eligible.length === 0) return 'No findings to fix right now.';
+  // Every real detector writes a recommendation, so an eligible finding with
+  // no route is a defensive case: still never call such a run clean.
+  if (steps.length === 0) return `${plural(eligible.length, 'finding')} to review`;
+  const lead = steps[0];
+  if (isIdleCapacityStep(lead) && facts.idlePct != null) return `Start with cluster size: ${facts.idlePct}% of executor capacity sat idle`;
+  if (lead.stageId != null) return `Start with Stage ${lead.stageId}`;
+  return `Start here: ${lowerFirst(findingActionLabel(lead.lead))}`;
+}
+
+/** The run-level summary under the title: how much was found and where, what
+ * the first fix is worth, and the run's idle capacity when that is large
+ * enough to matter but is not the first step (whose title already says it). */
+export function verdictSummary(eligible           , steps            , facts          )           {
+  const sentences           = [];
+  const { failedJobs, totalJobs } = facts.outcome;
+  if (failedJobs > 0) {
+    if (eligible.some((finding) => !FAILURE_TYPES.has(finding.type))) {
+      sentences.push('Fix the failure before tuning: the other findings cover only the work that ran.');
+    }
+  } else if (totalJobs > 0 && !facts.incomplete) {
+    sentences.push(totalJobs === 1 ? 'Its one job succeeded.' : `All ${totalJobs} jobs succeeded.`);
+  }
+  if (eligible.length === 0) {
+    if (failedJobs > 0) return sentences;
+    if (facts.clean) sentences.push('Every check passed for this run.');
+  } else if (steps.length === 0) {
+    sentences.push('They are listed by impact under Findings.');
+  } else {
+    sentences.push(`${plural(eligible.length, 'finding')} in ${plural(steps.length, 'place')}.`);
+    const wallClock = steps[0].lead.impactEstimate?.wallClock;
+    if (isIdleCapacityStep(steps[0])) {
+      sentences.push('A smaller cluster or dynamic allocation would free the idle cores for other jobs.');
+    } else if (wallClock && facts.runMs != null) {
+      sentences.push(`The first fix could save up to ${formatDuration(wallClock.high)} of this ${formatDuration(facts.runMs)} run.`);
+    }
+    if (steps.some((step) => step.related.length > 0)) {
+      sentences.push('Findings in the same stage usually share one cause, so they are grouped together and their savings overlap rather than add up.');
+    }
+  }
+  if (facts.incomplete) {
+    sentences.push('The log has no end-of-run record, so these figures cover only the part of the run it captured.');
+  }
+  const leadIsIdle = steps.length > 0 && isIdleCapacityStep(steps[0]);
+  if (!leadIsIdle && facts.idlePct != null && facts.idlePct >= IDLE_NOTABLE_PCT) {
+    sentences.push(
+      steps.some(isIdleCapacityStep)
+        ? `${facts.idlePct}% of the executor capacity sat idle, so the cluster may be larger than this job needs.`
+        : `${facts.idlePct}% of the run's core time went unused, so the cluster may be larger than this job needs.`,
+    );
+  }
+  return sentences;
+}
+
+const STACK_TRACE_HINT = 'Open the driver log only if you need the full stack trace.';
+
+/** What the failure step whose reason the verdict quotes tells the reader:
+ * the detector's "inspect the driver log for the reason" would send a
+ * newcomer looking for something already on screen. The copied text carries
+ * the reason itself, since "quoted above" means nothing once pasted. */
+export function quotedReasonText(reason        )                                    {
+  const sentence = /[.!?]$/.test(reason) ? reason : `${reason}.`;
+  return {
+    shown: `Spark's recorded reason is quoted above. ${STACK_TRACE_HINT}`,
+    copied: `Spark's recorded reason: ${sentence} ${STACK_TRACE_HINT}`,
+  };
+}
+
+/** One step as pasteable text: the action, what to try, and the savings. */
+export function stepCopyText(finding         , recommendation        , stageId                = null)         {
+  const impact = impactFigure(finding);
+  const meaning = savingsMeaning(finding);
+  const savings = impact && meaning ? `${impact} ${meaning}` : impact;
+  const where = stageId != null ? ` in Stage ${stageId}` : '';
+  const headline = `${findingActionLabel(finding)}${where}: ${recommendation}`;
+  return [/[.!?]$/.test(headline) ? headline : `${headline}.`, savings ? `Potential savings: ${savings}` : null]
+    .filter(Boolean)
+    .join(' ');
+}
+
+/** The whole verdict as a pasteable checklist for a ticket or a message:
+ * run, verdict, numbered steps (with their stage), and how many more places
+ * the full list holds. */
+export function planCopyText(input   
+                         
+                
+                                                      
+                    
+ )         {
+  const lines = [input.runName ? `Spark run ${input.runName}: ${input.title}` : input.title, ''];
+  input.steps.forEach(({ step, recommendation }, index) => {
+    lines.push(`${index + 1}. ${stepCopyText(step.lead, recommendation, step.stageId)}`);
+  });
+  if (input.remaining > 0) lines.push('', `${plural(input.remaining, 'more place')} to look at in the full findings list.`);
+  return lines.join('\n');
+}
+
+/** A step's "What to try" text for copying: Spark's quoted reason when it is this step's own
+ * failure, else the finding's recommendation. */
+export function stepCopyRecommendation(step          , outcome            )         {
+  return quotesReasonOf(step.lead, outcome) && outcome.reason
+    ? quotedReasonText(outcome.reason).copied
+    : recommendationText(step.lead);
+}
+
+/** Everything the verdict card shows, computed once for a run. `eligible` is what the verdict
+ * counts and ranks: the rollup-eligible findings of a known display type. */
+                                  
+                      
+                      
+                    
+                  
+                
+                    
+                                                                                       
+                    
+                    
+                                                                                
+                          
+ 
+
+export function buildRunVerdict(appModel          , allFindings           )                  {
+  const eligible = allFindings.filter((finding) => isEligible(finding) && DISPLAY_INDEX.has(finding.type));
+  const outcome = summarizeRunOutcome(appModel.jobs, allFindings);
+  const steps = buildNextSteps(eligible, outcome.failedJobs > 0 ? { failedJobStageIds: outcome.failedJobStageIds } : {});
+  const facts           = {
+    runMs: hasCompleteApplicationInterval(appModel.app) ? computeWallClock(appModel.app, appModel.stages).total : null,
+    idlePct: verdictIdlePct(steps, getScorecardEstimates(appModel).wastage.value),
+    incomplete: allFindings.some((finding) => finding.type === 'incompleteRun'),
+    clean: isCleanRun(appModel, allFindings),
+    outcome,
+    noFinishedStages: !hasFinishedStage(appModel.stages),
+  };
+  const title = verdictTitle(eligible, steps, facts);
+  const shown = steps.slice(0, NEXT_STEP_LIMIT);
+  const remaining = steps.length - shown.length;
+  const copyText = shown.length > 0
+    ? planCopyText({
+      runName: appModel.app?.name ?? null,
+      title,
+      steps: shown.map((step) => ({ step, recommendation: stepCopyRecommendation(step, outcome) })),
+      remaining,
+    })
+    : null;
+  return { eligible, outcome, steps, facts, title, summary: verdictSummary(eligible, steps, facts), shown, remaining, copyText };
+}

package/vendor-core/scaling-sim.js

@@ -6,7 +6,6 @@
 // Builds on computeCoreTimeSeries' clamp conceptually, but operates on the
 // worker-posted per-stage aggregates (runAggregates.perStage) so raw task data
 // never reaches the main thread.
-import { computeWallClock } from './wall-clock.js';
 import { computeTotalCores } from './core-count.js';
                                                                              
 
@@ -30,9 +29,11 @@ function estimatedTotalAtCores(perStage
   return sum;
 }
 
-export function simulateScaling({ app, stages, runAggregates, executorsAdded }   
+/** `observedActiveMs` is the run's stages-active wall-clock (the interpretation's
+ * `wallClock.stagesActive`), the one observed makespan predictions are scaled against. */
+export function simulateScaling({ app, observedActiveMs, runAggregates, executorsAdded }   
+                           
                            
-                               
                                       
                                   
  )   
@@ -48,8 +49,6 @@ export function simulateScaling({ app, stages, runAggregates, executorsAdded }
   // `executorsAdded` cast: computeTotalCores only reads `totalCores`, present on
   // ExecutorAddedEvent (the only kind passed here) but not on the union type.
   const baselineCores = computeTotalCores(app ?? {}, executorsAdded                                  );
-  const wc = computeWallClock(app, stages                                                               );
-  const observedActiveMs = wc.stagesActive;
 
   // Model Error: predicted-at-baseline vs. observed stages-active wall-clock.
   const predictedAtBaseline = baselineCores > 0 ? estimatedTotalAtCores(perStage, baselineCores) : 0;

package/vendor-core/scorecard-estimates.js

@@ -0,0 +1,62 @@
+import { computeEfficiencyModel } from './efficiency-model.js';
+import { computeWallClock } from './wall-clock.js';
+                                           
+
+                           
+                       
+                                                 
+  
+
+                        
+                       
+                                                                                              
+  
+
+export function hasCompleteApplicationInterval(app                 )          {
+  return typeof app?.startTime === 'number'
+    && Number.isFinite(app.startTime)
+    && typeof app?.endTime === 'number'
+    && Number.isFinite(app.endTime)
+    && app.endTime > app.startTime;
+}
+
+export function getScorecardEstimates(appModel          )   
+                                 
+                           
+  {
+  if (!hasCompleteApplicationInterval(appModel.app)) {
+    return {
+      efficiency: { value: null, unavailableReason: 'application-timing' },
+      wastage: { value: null, unavailableReason: 'application-timing' },
+    };
+  }
+
+  const wallClock = computeWallClock(appModel.app, appModel.stages);
+  const efficiency = Math.min(100, Math.round((wallClock.stagesActive / wallClock.total) * 100));
+
+  if (!appModel.runAggregates) {
+    return {
+      efficiency: { value: efficiency, unavailableReason: null },
+      wastage: { value: null, unavailableReason: 'core-usage-summary' },
+    };
+  }
+
+  const efficiencyModel = computeEfficiencyModel({
+    app: appModel.app,
+    stages: appModel.stages,
+    executorsAdded: appModel.executors.added,
+    runAggregates: appModel.runAggregates,
+  });
+
+  if (!Number.isFinite(efficiencyModel.availableComputeHours) || efficiencyModel.availableComputeHours <= 0) {
+    return {
+      efficiency: { value: efficiency, unavailableReason: null },
+      wastage: { value: null, unavailableReason: 'executor-capacity' },
+    };
+  }
+
+  return {
+    efficiency: { value: efficiency, unavailableReason: null },
+    wastage: { value: efficiencyModel.wastagePct, unavailableReason: null },
+  };
+}