sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/evidence-report.js

@@ -2,19 +2,33 @@
 // then serializes a run summary + findings into a deterministic, byte-stable JSON + Markdown.
 // Raw task records are never included (privacy baseline); redaction is opt-in via { redact: true }.
 import { analyze, auditConfig } from './analyzer.js';
-import { detectorCatalog } from './detectors.js';
-import { typeTag, formatBytes, formatDuration, IMPACT_BAND_ORDER } from './format-utils.js';
-import { FINDING_NAMES, titleCase } from './finding-names.js';
-import { redactReport } from './redact.js';
-import { coreFindingActionLabel } from './finding-action-label.js';
-import { matchesFindingFilterCriteria } from './finding-filter-predicate.js';
-import { buildRecommendationRollup, isEligible, rankFindings,                  } from './recommendation-rollup.js';
+                                                         
+import {
+  describeTunedThresholds, tunedDetectorCatalog, tunedDetectors, tunedRunNote, tunedThresholdsForType,
+} from './threshold-overrides.js';
 import { getThresholdSummary } from './threshold-summary.js';
+import {
+  typeTag, formatBytes, formatCores, formatDuration, formatRawWaste, formatWallClockRange, IMPACT_BAND_ORDER, readsAsZero,
+} from './format-utils.js';
+import { findingName, titleCase } from './finding-names.js';
+import { redactReport, redactRunModel } from './redact.js';
+import { formatTaskFailureHeadline,                       } from './task-failure.js';
+import { findingActionLabel } from './finding-action-label.js';
+import { matchesFindingFilterCriteria, singleStageId } from './finding-filter-predicate.js';
+import { buildRecommendationRollup, isEligible, isRealFinding, rankFindings,                  } from './recommendation-rollup.js';
+import { checkCoverage, isCleanRun } from './check-coverage.js';
+import { buildRunVerdict, stepCopyRecommendation, stepCopyText,                      } from './run-verdict.js';
+import {
+  estimateProvenance, impactEstimateFigure, impactFigure, rawWasteMeaning, savingsMeaning,
+} from './impact-format.js';
+import { computeRunShape,               } from './run-shape.js';
+import { detectorInfoByType } from './detector-docs.js';
              
-                                                                                                    
+                                                                                                                     
+                  
                     
 
-export const EVIDENCE_SCHEMA_VERSION         = 3;
+export const EVIDENCE_SCHEMA_VERSION         = 5;
 
 // findingRow always sets id/metric/value/recommendation via `?? null` (never omits the key), and
 // buildJson does the same for evidenceAvailability and summary.app.{id,name,sparkVersion}: these
@@ -22,20 +36,34 @@ export const EVIDENCE_SCHEMA_VERSION         = 3;
 // "unknown version" sentinel). Kept as `?? null`, not `?? undefined`: JSON.stringify drops
 // undefined keys but keeps null, so undefined would silently strip these from the report.
 //
-// `value` is number|string|null: stageFailed and the configAudit entries put text in Finding.value
-// instead of a magnitude, carried through unchanged.
+// `value` is always a magnitude or null. The text-valued findings (stageFailed's failure reason,
+// configAudit's current setting, incompleteRun's 'missing') carry theirs in `valueText` instead,
+// present only on those rows.
                              
-                                                                                                      
-                                                                                                                 
-                                                             
+                                                                                        
+                                                                                            
+                                                          
                                                                                                   
                                                                                                  
                                                               
                       
                                                                        
                                   
+                                                                                                 
+                                                                                 
+                  
+                                
+                                                                                       
+                                                                                   
+                                    
  
 
+// One row per finding, discriminated on `type`: `evidence` is that type's public evidence
+// (FindingEvidenceMap, projected through EVIDENCE_KEYS below), never the finding's other fields.
+                          
+                                                                                       
+               
+
 // The `Fix these first` rollup row: one entry per buildRecommendationRollup
 // group, so the CLI/MCP/download paths get the same impact-ranked aggregation the dashboard shows.
                                     
@@ -50,75 +78,185 @@ export const EVIDENCE_SCHEMA_VERSION         = 3;
                       
                  
                                                      
+                                                                                                 
+                                                                                                  
+                                                 
                         
+                               
  
 
-// One line per detector `type` that fired zero findings, so a flat report can state "these were
-// checked and came back clean" like the dashboard's clean-checks table.
+// One line per detector `type` that fired zero findings and could run, so a flat report can state
+// "these were checked and came back clean" like the dashboard's clean-checks table.
                                   
                
               
                            
+                                                                                                  
+                                    
+ 
+
+// A detector `type` with zero findings that the log lacked the data to run (the dashboard's "Not
+// checked on this log" group), so it is not reported as passed. `reason` says why, naming the
+// setting to turn on where the detector gives one.
+                                                           
+                 
+ 
+
+                                                                            
+
+// How the run ended, as far as its jobs say (run-outcome.ts, the same summary the dashboard's
+// verdict leads with). Counts only jobs with an end record; `failureReason` is the first line of
+// Spark's own recorded reason, only when a job failed, and `failureReasonStageId` the stage it came
+// from (null when there is no reason or it came from a job's exception).
+// One verdict step: a place to look (a stage, or an app-level problem), led by its best-ranked
+// finding, with the other finding types flagged there. `text` is the step's line in the
+// dashboard's "Copy next steps" checklist.
+                                 
+              
+                         
+               
+              
+                               
+                      
+                         
+                        
+                               
+                         
+               
+ 
+
+// The dashboard's run verdict (run-verdict.ts): title, summary sentences, the first steps in the
+// same order, how many more places the full list holds, and the "Copy next steps" text.
+                              
+                
+                    
+                          
+                          
+                          
+ 
+
+                                    
+                     
+                    
+                               
+                                      
  
 
                                      
                         
             
                                                                                     
-                                                                                          
-                                                                          
+                                                                    
+                                                                                         
+                         
+                                       
+                                                                                                  
+                            
+                                   
+                                                 
+                                                                                                   
+                   
+                               
+                       
+                                                                                                   
+                                                      
     
+                       
                                                     
                      
                          
                                        
                                  
+                                   
  
 
-// Fields surfaced as first-class report columns. Everything else on a finding
-// becomes its `evidence` payload (sorted for stable key order).
-const CORE_KEYS = new Set([
-  'id', 'type', 'name', 'impactBand', 'stageId', 'metric', 'value',
-  'recommendation', 'detectorVersion', 'confidence', 'validationRequired', 'docAnchor', 'impactEstimate',
-  'actionLabel',
-]);
-
-// Internal-only fields with no meaning to a human reading this report: never surfaced as a core
-// column, and also excluded from the generic evidence dump (unlike stageIds, which IS actionable
-// to a reader). `planNodeIds` is view-layer plan-graph node ids (Plan Advisor detectors, see
-// plan-graph-model.ts): on a real log it can carry a hundred-plus ids, which would otherwise print
-// as one unreadable `- planNodeIds: [...]` line and bloat the report for no reader benefit.
-const NON_EVIDENCE_KEYS = new Set(['planNodeIds']);
+// Each finding type's public evidence fields: exactly the keys of its FindingEvidenceMap entry
+// (finding-types.ts), checked both ways at compile time. A field a detector adds for another core
+// module (stageShape's totalCores, utilization's unrounded fraction) is left off both, so it never
+// reaches the report; adding, renaming or dropping a key here changes the report contract.
+const EVIDENCE_KEYS = {
+  skew: [],
+  stageShape: ['rule'],
+  shuffle: [],
+  partitionSizing: ['rule'],
+  spill: ['spillMagnitude'],
+  gc: ['direction'],
+  slowHost: ['variant', 'host', 'hostTaskShare', 'hostMeanMs', 'dimension', 'executorId', 'execMaxValue'],
+  stageSlowness: [],
+  stageFailed: ['variant', 'numTasks', 'memoryBytesSpilled', 'failedTaskDetails'],
+  failures: ['failedTasks', 'dominantReason', 'dominantError', 'failureGroups', 'otherFailedTasks'],
+  straggler: ['unit', 'speculativeTasks', 'stragglerCount'],
+  speculationWaste: [],
+  retryWaste: ['numTasks', 'memoryBytesSpilled', 'retriedTaskDetails'],
+  tinyTask: [],
+  incompleteRun: [],
+  coldStart: [],
+  utilization: ['cpuUtilizationPct'],
+  memoryUtilization: ['variant', 'rule', 'executorId', 'heap', 'dataUnavailable'],
+  cacheUtilization: [
+    'variant', 'rddId', 'rddName', 'memorySize', 'diskSize', 'numCachedPartitions', 'numPartitions', 'dataUnavailable',
+  ],
+  coreLocality: ['nonLocalTaskCount'],
+  autoscalingChurn: ['shortLivedExecutorCount'],
+  cachingOpportunity: ['variant', 'relation', 'format', 'relations', 'operator', 'executionIds', 'totalReadBytes'],
+  jobFailureRate: ['failedJobs', 'totalJobs', 'failedTasks', 'totalTasks', 'avgJobDurationMs', 'taskFailureRate'],
+  configAudit: ['property'],
+  duplicatePlanSubtree: [
+    'executionId', 'stageIds', 'stageShares', 'occurrencesIdentical', 'rootName', 'subtreeSize', 'sampleRelation',
+    'groupIndex',
+  ],
+  smallFiles: ['executionId', 'stageIds', 'fileCount', 'direction', 'nodeName'],
+  underBroadcast: ['executionId', 'stageIds', 'largerSideBytes'],
+  overBroadcast: ['executionId', 'stageIds'],
+}                                                                                    ;
 
-function findingRow(f         )             {
+// The other direction: an evidence field EVIDENCE_KEYS doesn't list fails here.
+                            
+                                                                                              
+               
+                                      
+                                                                    
+
+// The finding's evidence, keys sorted for a stable order. An undefined field (spill's
+// spillMagnitude without a magnitude) is absent, as in the JSON.
+function projectEvidence(f         )                          {
+  const fields = f                                      ;
   const evidence                          = {};
-  for (const k of Object.keys(f).sort()) {
-    // An undefined field (spill's spillMagnitude without a magnitude) is absent, as in the JSON.
-    const v = (f                           )[k];
-    if (!CORE_KEYS.has(k) && !NON_EVIDENCE_KEYS.has(k) && v !== undefined) evidence[k] = v;
+  for (const k of [...EVIDENCE_KEYS[f.type]].sort()) {
+    if (fields[k] !== undefined) evidence[k] = fields[k];
   }
-  const row             = {
+  return evidence;
+}
+
+function findingRow(f         )             {
+  // Cast: projectEvidence's keys come from EVIDENCE_KEYS[f.type], so `evidence` is that type's
+  // FindingEvidenceMap entry, which TypeScript can't correlate with `type` on its own.
+  const row = {
     id: f.id ?? null,
     type: f.type,
-    name: titleCase(FINDING_NAMES[f.type] ?? f.type),
+    name: titleCase(findingName(f.type)),
     tag: typeTag(f.type),
     impactBand: f.impactBand,
     stageId: f.stageId ?? null,
     metric: f.metric ?? null,
     value: f.value ?? null,
+    ...(f.valueText != null ? { valueText: f.valueText } : {}),
     recommendation: f.recommendation ?? null,
     detectorVersion: f.detectorVersion ?? 1,
-    evidence,
-    // Deliberate simplification vs the view layer's REGISTRY fallback: no widget registry here, and
-    // falling back to the finding's own `type` is fine since coreFindingActionLabel already covers
-    // every emitted type; only obscure/future sub-variants hit this fallback.
-    actionLabel: coreFindingActionLabel(f) ?? f.type,
-  };
+    evidence: projectEvidence(f),
+    actionLabel: findingActionLabel(f),
+  }              ;
   // Threshold/confidence provenance, only when the detector emitted it.
   if (f.confidence != null) row.confidence = f.confidence;
   if (f.validationRequired != null) row.validationRequired = f.validationRequired;
   if (f.docAnchor != null) row.docAnchor = f.docAnchor;
   if (f.impactEstimate != null) row.impactEstimate = f.impactEstimate;
+  const figure = impactEstimateFigure(f.impactEstimate);
+  if (figure) {
+    row.impact = figure.text;
+    row.impactMeaning = figure.meaning;
+  }
+  if (f.tunedThresholds != null) row.tunedThresholds = f.tunedThresholds;
   return row;
 }
 
@@ -156,7 +294,7 @@ function buildRecommendations(
     const base = {
       type: group.type,
       tag: typeTag(group.type),
-      actionLabel: coreFindingActionLabel(representative) ?? representative.type,
+      actionLabel: findingActionLabel(representative),
       findingCount: group.findingCount,
       findingIds,
     };
@@ -169,15 +307,19 @@ function buildRecommendations(
         // A point estimate, not a range: matches FixTheseFirst.tsx's trailingStat for time groups,
         // which prints the same figure twice rather than the finding-level spread computeStageUnionMs collapsed.
         impact: formatWallClockRange(group.recoverableMsHigh, group.recoverableMsHigh),
+        impactMeaning: 'of run time',
       };
     }
     if (group.kind === 'resource') {
+      const text = formatRawWaste({ value: group.total, unit: group.unit });
+      const shown = readsAsZero(text) ? null : text;
       return {
         ...base,
         kind: 'resource',
         unit: group.unit,
         total: group.total,
-        impact: formatRawWaste({ value: group.total, unit: group.unit }),
+        impact: shown,
+        impactMeaning: shown ? rawWasteMeaning(group.unit) : null,
       };
     }
     return {
@@ -187,51 +329,138 @@ function buildRecommendations(
       // No single quantifiable figure for a count group; the impact-band tally
       // (byImpactBand above) is the payload instead.
       impact: null,
+      impactMeaning: null,
     };
   });
 }
 
-// Detector types that fired zero findings, so a flat report can state "checked and clean". Differs
-// from the dashboard's Alerts.tsx "Clean checks", which excludes coreLocality (the one remaining
-// always-mounted reference widget, shown elsewhere); a flat report has no such separate surface,
-// so this includes it too when it has zero findings.
-function buildCleanChecks(findings           )                    {
-  const firedTypes = new Set(findings.map((f) => f.type));
-  const seen = new Set        ();
-  const entries                    = [];
-  // detectorCatalog() can list the same type more than once (configAudit has 4 entries); dedupe by
-  // type, keeping first, so a type with sibling entries contributes exactly one clean-check line.
-  for (const d of detectorCatalog()                           ) {
-    if (firedTypes.has(d.type) || seen.has(d.type)) continue;
-    seen.add(d.type);
-    entries.push({ type: d.type, tag: typeTag(d.type), thresholdSummary: getThresholdSummary(d.type) });
+// Finding types with no real finding, split into those that passed and those the log could not
+// run (the rule the dashboard's Clean checks uses, from check-coverage.ts). Differs from
+// Alerts.tsx in one way: the dashboard excludes coreLocality (the one always-mounted reference
+// widget, shown elsewhere); a flat report has no such separate surface, so this includes it too.
+function buildCheckLists(
+  findings           , stages                    , thresholds                                ,
+)                                                                       {
+  // isRealFinding: a type whose only finding is an evidence caveat (memoryUtilization's
+  // dataUnavailable variant) had nothing to check, so it lands in notRunChecks.
+  const firedTypes = new Set        (findings.filter(isRealFinding).map((f) => f.type));
+  const coverage = checkCoverage(stages, findings);
+  const cleanChecks                    = [];
+  const notRunChecks                     = [];
+  // One line per emitted finding type (configAudit's four entries give one line;
+  // broadcastSizing gives overBroadcast and underBroadcast), the same set the dashboard lists.
+  for (const [type, { thresholdSummary }] of Object.entries(detectorInfoByType())) {
+    if (firedTypes.has(type)) continue;
+    // A tuned check was measured against the tuned criterion, so it says which one.
+    const tuned = tunedThresholdsForType(type, thresholds);
+    const entry                  = tuned
+      ? { type, tag: typeTag(type), thresholdSummary: getThresholdSummary(type, thresholds), tunedThresholds: tuned }
+      : { type, tag: typeTag(type), thresholdSummary };
+    const reason = coverage.notRunReason(type);
+    if (reason) notRunChecks.push({ ...entry, reason });
+    else cleanChecks.push(entry);
   }
-  return entries;
+  return { cleanChecks, notRunChecks };
+}
+
+function countByImpactBand(findings                               )                   {
+  const counts                   = { critical: 0, warning: 0, info: 0 };
+  for (const f of findings) if (f.impactBand in counts) counts[f.impactBand                          ] += 1;
+  return counts;
+}
+
+function verdictJson(model                 )              {
+  return {
+    title: model.title,
+    summary: model.summary,
+    steps: model.shown.map((step)                 => {
+      const recommendation = stepCopyRecommendation(step, model.outcome);
+      return {
+        key: step.key,
+        stageId: step.stageId,
+        type: step.lead.type,
+        tag: typeTag(step.lead.type),
+        leadFindingId: step.lead.id ?? null,
+        actionLabel: findingActionLabel(step.lead),
+        recommendation,
+        impact: impactFigure(step.lead),
+        impactMeaning: savingsMeaning(step.lead),
+        relatedTypes: step.related.map((f) => f.type),
+        text: stepCopyText(step.lead, recommendation, step.stageId),
+      };
+    }),
+    remainingPlaces: model.remaining,
+    copyText: model.copyText,
+  };
 }
 
 // Keyed by appModel object identity: mcp-tools.ts caches one fixed appModel per runId (never
 // mutated), so re-running analyze()/auditConfig() reproduces the same catalog. getFindingEvidence
 // calls buildEvidenceReport once per drill-down; without this, N lookups meant N detector re-runs.
 // A WeakMap needs no invalidation: once mcp-tools.ts evicts the appModel, this entry is collectible.
-const jsonCache = new WeakMap                              ();
+// Each cache is split first by the overrides object the report ran under (one fixed, frozen object
+// per CLI invocation or MCP server process; DEFAULT_THRESHOLDS for the specification's).
+                                                                          
+const DEFAULT_THRESHOLDS = {};
+const jsonCache              = new WeakMap();
+// The redacted report, keyed by the unredacted appModel it was built from.
+const redactedJsonCache              = new WeakMap();
 
-function buildJson(appModel          )                     {
-  const cached = jsonCache.get(appModel);
-  if (cached) return cached;
-  const { app, stages, executors, sql, jobs, runAggregates, evidenceAvailability } = appModel;
+function cacheFor(cache             , thresholds                                )                                        {
+  const key = thresholds ?? DEFAULT_THRESHOLDS;
+  let byModel = cache.get(key);
+  if (!byModel) {
+    byModel = new WeakMap();
+    cache.set(key, byModel);
+  }
+  return byModel;
+}
+
+function runFindings(appModel          , thresholds                                )                                            {
+  const { app, stages, executors, sql, jobs, runAggregates } = appModel;
   const catalog = analyze(
     app, stages, executors?.added ?? [], executors?.removed ?? [],
     jobs ?? new Map(), sql ?? new Map(),
-    runAggregates ?? null,
+    runAggregates ?? null, { thresholds },
   );
-  const config = auditConfig(app);
+  return { catalog, config: auditConfig(app) };
+}
+
+// Redacts the model and findings before the report derives any text from them, the same order
+// the HTML export uses: the verdict truncates Spark's failure reason, and redacting that
+// truncated copy afterwards would miss an identifier the cut left as a fragment.
+function buildRedactedJson(appModel          , thresholds                                )                     {
+  const cache = cacheFor(redactedJsonCache, thresholds);
+  const cached = cache.get(appModel);
+  if (cached) return cached;
+  const { catalog, config } = runFindings(appModel, thresholds);
+  const run = redactRunModel(appModel, catalog, config);
+  // redactReport stays as a last pass: idempotent over pseudonyms, and it covers the report's own
+  // structured fields (summary.app.id) the same way it always has.
+  const result = redactReport(buildJson(run.appModel, thresholds, { catalog: run.catalog, config: run.configFindings }));
+  cache.set(appModel, result);
+  return result;
+}
+
+function buildJson(
+  appModel          , thresholds                                , findings                                            ,
+)                     {
+  const cache = cacheFor(jsonCache, thresholds);
+  const cached = cache.get(appModel);
+  if (cached) return cached;
+  const { app, stages, executors, sql, jobs, evidenceAvailability } = appModel;
+  const { catalog, config } = findings ?? runFindings(appModel, thresholds);
   const allFindings = [...catalog, ...config];
   const rows = sortFindings(allFindings.map(findingRow));
   const recommendations = buildRecommendations(allFindings, stages ?? new Map());
-  const cleanChecks = buildCleanChecks(allFindings);
-
-  const impactBandCounts = { critical: 0, warning: 0, info: 0 };
-  for (const r of rows) if (r.impactBand in impactBandCounts) impactBandCounts[r.impactBand] += 1;
+  const { cleanChecks, notRunChecks } = buildCheckLists(allFindings, stages ?? new Map(), thresholds);
+  const tuned = tunedDetectors(thresholds);
+  const actionable = allFindings.filter(isEligible);
+  const fullModel           = {
+    ...appModel, stages: stages ?? new Map(), jobs: jobs ?? new Map(), executors: executors ?? { added: [], removed: [] },
+  };
+  const runVerdict = buildRunVerdict(fullModel, allFindings);
+  const runOutcome = runVerdict.outcome;
 
   const result                     = {
     schemaVersion: EVIDENCE_SCHEMA_VERSION,
@@ -247,17 +476,30 @@ function buildJson(appModel          )                     {
       jobCount: jobs?.size ?? 0,
       sqlExecutionCount: sql?.size ?? 0,
       findingCount: rows.length,
-      impactBandCounts,
+      impactBandCounts: countByImpactBand(rows),
+      actionableFindingCount: actionable.length,
+      actionableImpactBandCounts: countByImpactBand(actionable),
+      clean: isCleanRun({ jobs: jobs ?? new Map(), stages: stages ?? new Map() }, allFindings),
+      outcome: {
+        failedJobs: runOutcome.failedJobs,
+        totalJobs: runOutcome.totalJobs,
+        failureReason: runOutcome.reason,
+        failureReasonStageId: runOutcome.reason != null ? runOutcome.reasonStageId : null,
+      },
+      runShape: computeRunShape(fullModel),
+      ...(tuned ? { tunedThresholds: tuned } : {}),
     },
+    verdict: verdictJson(runVerdict),
     evidenceAvailability: evidenceAvailability ?? null,
     // Detector metadata so the threshold set that produced each finding travels with the evidence.
     // Order follows DETECTORS (stable) => byte-stable serialization.
-    detectors: detectorCatalog(),
+    detectors: tunedDetectorCatalog(thresholds),
     findings: rows,
     recommendations,
     cleanChecks,
+    notRunChecks,
   };
-  jsonCache.set(appModel, result);
+  cache.set(appModel, result);
   return result;
 }
 
@@ -269,43 +511,107 @@ function renderEvidenceValue(key        , value         )         {
   return String(value);
 }
 
-function formatWallClockRange(low        , high        )         {
-  const fmtMs = (ms        ) => (ms === 0 ? '0s' : formatDuration(ms));
-  return low === high ? `Estimated ${fmtMs(high)}` : `Estimated ${fmtMs(low)}-${fmtMs(high)}`;
+// The `failures` finding's distinct errors: a headline per group, its stack excerpt as an indented
+// code block (indented, not fenced, so no excerpt content can close it early).
+function renderFailureGroups(groups                    )           {
+  const lines = [`  - failureGroups: ${groups.length}`];
+  for (const g of groups) {
+    lines.push(`    - ${g.count} task(s): ${formatTaskFailureHeadline(g)}`);
+    if (g.stackExcerpt) {
+      lines.push('');
+      for (const l of g.stackExcerpt.split('\n')) lines.push(`          ${l}`);
+      lines.push('');
+    }
+  }
+  return lines;
+}
+
+// The finding's "Potential savings" figure as the dashboard shows it (impactEstimateFigure: the
+// range, or the raw waste only without a range, nothing for a zero or informational estimate),
+// followed by what it counts. `basis: 'informational'` findings carry nothing to print.
+function renderImpactEstimate(estimate                )                {
+  const figure = impactEstimateFigure(estimate);
+  if (!figure) return null;
+  return `${figure.text}${figure.meaning ? ` ${figure.meaning}` : ''} (estimateMethod: ${estimate.estimateMethod})`;
 }
 
-function formatRawWaste(rawWaste                )         {
-  const rounded = Math.round(rawWaste.value * 10) / 10;
-  switch (rawWaste.unit) {
-    case 'bytes': return formatBytes(rawWaste.value);
-    case 'ms': return formatDuration(rawWaste.value);
-    case 'mbSeconds': return `${rounded} MB-s`;
-    case 'coreHours': return `${rounded.toFixed(1)} core-h`;
-    case 'coreMs': return `${rounded} core-ms`;
-    default: return String(rawWaste.value);
+// The run's job results in one line, worded like the dashboard verdict: failures first, with
+// Spark's recorded reason; null when the log records no ended job.
+function renderOutcome(outcome                   , incomplete         )                {
+  const { failedJobs, totalJobs, failureReason, failureReasonStageId } = outcome;
+  if (totalJobs === 0) return null;
+  if (failedJobs > 0) {
+    const failed = failedJobs < totalJobs
+      ? `${failedJobs} of ${totalJobs} jobs failed.`
+      : totalJobs === 1 ? 'The run\'s one job failed.' : `All ${totalJobs} jobs failed.`;
+    if (!failureReason) return failed;
+    const where = failureReasonStageId != null ? ` (stage ${failureReasonStageId})` : '';
+    return `${failed} Spark's recorded reason${where}: ${failureReason}`;
   }
+  const succeeded = totalJobs === 1 ? 'Its one job succeeded.' : `All ${totalJobs} jobs succeeded.`;
+  return incomplete ? `${succeeded} The log has no end-of-run record, so jobs still running when it stops are not counted.` : succeeded;
 }
 
-// `basis: 'informational'` findings carry no wallClock/rawWaste at all, so
-// there's nothing quantifiable to print; the caller skips the line entirely.
-function renderImpactEstimate(estimate                )                {
-  const rangeText = estimate.wallClock ? formatWallClockRange(estimate.wallClock.low, estimate.wallClock.high) : null;
-  const wasteText = estimate.rawWaste ? formatRawWaste(estimate.rawWaste) : null;
-  if (!rangeText && !wasteText) return null;
-  const parts = [rangeText, wasteText].filter((p)              => p != null).join(' · ');
-  return `${parts} (estimateMethod: ${estimate.estimateMethod})`;
+// The Scorecard, ETL phases and core-usage figures, each worded to say what it measures, since
+// "efficiency" and "unused core time" are different shares. A figure the dashboard cannot show is
+// left out.
+function renderRunShape(shape          )           {
+  const lines           = [];
+  if (shape.wallClockMs != null) lines.push(`- Wall-clock: ${formatDuration(shape.wallClockMs)}`);
+  if (shape.efficiencyPct != null) lines.push(`- Efficiency: ${shape.efficiencyPct}% (share of the run with a stage running)`);
+  if (shape.unusedCoreTimePct != null) {
+    lines.push(`- Unused core time: ${shape.unusedCoreTimePct}% (driver idle plus executor slack, as a share of available core time)`);
+  }
+  if (shape.peakBusyCores != null) lines.push(`- Peak busy cores: ${formatCores(shape.peakBusyCores)}`);
+  if (shape.etlPhasesMs) {
+    const { extract, transform, load } = shape.etlPhasesMs;
+    lines.push(`- ETL phases (summed stage time): extract ${formatDuration(extract)}, transform ${formatDuration(transform)}, load ${formatDuration(load)}`);
+  }
+  return lines;
 }
 
-function renderMarkdown(json                    )         {
-  const { summary, findings, evidenceAvailability, detectors, recommendations, cleanChecks } = json;
+// The dashboard verdict card as text: title, summary, then the numbered steps worded as its
+// "Copy next steps" checklist, each followed by the other finding types flagged at that place.
+function renderVerdict(verdict             )           {
+  const lines = ['## Verdict', '', verdict.title];
+  if (verdict.summary.length > 0) lines.push('', verdict.summary.join(' '));
+  if (verdict.steps.length > 0) {
+    lines.push('');
+    verdict.steps.forEach((step, i) => {
+      lines.push(`${i + 1}. [${step.tag}] ${step.text}`);
+      if (step.relatedTypes.length > 0) {
+        const related = step.relatedTypes.map(findingName).join(', ');
+        lines.push(`   - Also flagged here: ${related}. These often share this cause, so the same fix may clear them too.`);
+      }
+    });
+    if (verdict.remainingPlaces > 0) {
+      const places = `${verdict.remainingPlaces} more place${verdict.remainingPlaces === 1 ? '' : 's'}`;
+      lines.push('', `${places} to look at in the Findings section below.`);
+    }
+  }
+  lines.push('');
+  return lines;
+}
+
+// `incomplete` comes from the unfiltered findings, since a findingsFilter can drop the incompleteRun row.
+function renderMarkdown(json                    , incomplete         )         {
+  const { summary, verdict, findings, evidenceAvailability, detectors, recommendations, cleanChecks, notRunChecks } = json;
   const lines           = [];
   lines.push('# Spark run evidence report');
   lines.push('');
   lines.push(`- Application: ${summary.app.name ?? '(unknown)'} (${summary.app.id ?? 'n/a'})`);
   lines.push(`- Spark version: ${summary.app.sparkVersion ?? 'n/a'}`);
+  if (summary.tunedThresholds) lines.push(`- Tuned thresholds: ${tunedRunNote(summary.tunedThresholds)}`);
   lines.push(`- Stages: ${summary.stageCount} · Jobs: ${summary.jobCount} · SQL executions: ${summary.sqlExecutionCount}`);
   lines.push(`- Findings: ${summary.findingCount} (critical ${summary.impactBandCounts.critical}, warning ${summary.impactBandCounts.warning}, info ${summary.impactBandCounts.info})`);
+  const actionableCounts = summary.actionableImpactBandCounts;
+  lines.push(`- Findings to act on: ${summary.actionableFindingCount} (critical ${actionableCounts.critical}, warning ${actionableCounts.warning}, info ${actionableCounts.info})`);
+  const outcomeLine = renderOutcome(summary.outcome, incomplete);
+  if (outcomeLine) lines.push(`- Outcome: ${outcomeLine}`);
+  if (summary.clean) lines.push('- Clean run: no findings, no failed jobs, and every check could run.');
+  lines.push(...renderRunShape(summary.runShape));
   lines.push('');
+  lines.push(...renderVerdict(verdict));
   if (recommendations.length > 0) {
     lines.push(`## Fix these first (${recommendations.length})`);
     lines.push('');
@@ -313,8 +619,8 @@ function renderMarkdown(json                    )         {
       lines.push(`${i + 1}. [${r.tag}] ${r.actionLabel}`);
       const detail = r.kind === 'count'
         ? Object.entries(r.byImpactBand ?? {}).map(([impactBand, count]) => `${count} ${impactBand}`).join(', ')
-        : r.impact;
-      lines.push(`   - ${detail} · ×${r.findingCount} finding(s)`);
+        : r.impact && `${r.impact}${r.impactMeaning ? ` ${r.impactMeaning}` : ''}`;
+      lines.push(`   - ${detail ? `${detail} · ` : ''}×${r.findingCount} finding(s)`);
     });
     lines.push('');
   }
@@ -324,19 +630,25 @@ function renderMarkdown(json                    )         {
     const where = r.stageId != null ? ` (stage ${r.stageId})` : '';
     lines.push(`### ${r.name} · ${r.impactBand}${where}`);
     lines.push(`- action: ${r.actionLabel}`);
-    if (r.metric != null) lines.push(`- ${r.metric}: ${r.value}`);
+    if (r.metric != null) lines.push(`- ${r.metric}: ${r.valueText ?? r.value}`);
     if (r.recommendation) lines.push(`- ${r.recommendation}`);
     if (r.confidence) lines.push(`- confidence: ${r.confidence}`);
     if (r.validationRequired) lines.push(`- validation: ${r.validationRequired}`);
+    if (r.tunedThresholds) lines.push(`- tuned thresholds: ${describeTunedThresholds(r.tunedThresholds)}`);
     const impactText = r.impactEstimate ? renderImpactEstimate(r.impactEstimate) : null;
     if (impactText) lines.push(`- impact: ${impactText}`);
+    const provenance = r.impactEstimate ? estimateProvenance(r) : null;
+    if (provenance) lines.push(`- estimate: ${provenance}`);
     lines.push(`- detector version: ${r.detectorVersion}`);
     // Evidence payload (sorted for stable order) so two rows differing only by evidence (two
     // smallFiles by direction, two partitionSizing by rule) render distinctly.
     const evidence = Object.entries(r.evidence ?? {}).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
     if (evidence.length) {
       lines.push('- evidence:');
-      for (const [k, v] of evidence) lines.push(`  - ${k}: ${renderEvidenceValue(k, v)}`);
+      for (const [k, v] of evidence) {
+        if (k === 'failureGroups' && Array.isArray(v)) lines.push(...renderFailureGroups(v                      ));
+        else lines.push(`  - ${k}: ${renderEvidenceValue(k, v)}`);
+      }
     }
     lines.push('');
   }
@@ -353,7 +665,18 @@ function renderMarkdown(json                    )         {
     lines.push('## Detectors');
     lines.push('');
     for (const d of detectors) {
-      lines.push(`- ${d.type} (v${d.version}, ${d.scope}), thresholds: ${JSON.stringify(d.thresholds)}`);
+      const tuned = d.tunedThresholds ? ` (tuned: ${describeTunedThresholds(d.tunedThresholds)})` : '';
+      lines.push(`- ${d.type} (v${d.version}, ${d.scope}), thresholds: ${JSON.stringify(d.thresholds)}${tuned}`);
+    }
+    lines.push('');
+  }
+  if (notRunChecks.length > 0) {
+    lines.push(`## Not checked on this log (${notRunChecks.length})`);
+    lines.push('');
+    lines.push('The log lacked the data these checks need, so they neither passed nor failed.');
+    lines.push('');
+    for (const c of notRunChecks) {
+      lines.push(`- [${c.tag}] ${c.type}: ${c.reason}`);
     }
     lines.push('');
   }
@@ -361,7 +684,8 @@ function renderMarkdown(json                    )         {
     lines.push(`## Clean checks (${cleanChecks.length})`);
     lines.push('');
     for (const c of cleanChecks) {
-      lines.push(`- [${c.tag}] ${c.type}: ${c.thresholdSummary}`);
+      const tuned = c.tunedThresholds ? ` (tuned: ${describeTunedThresholds(c.tunedThresholds)})` : '';
+      lines.push(`- [${c.tag}] ${c.type}: ${c.thresholdSummary}${tuned}`);
     }
     lines.push('');
   }
@@ -377,7 +701,10 @@ function renderMarkdown(json                    )         {
 // CLI/MCP-facing filter over FindingRow, delegating to the shared core predicate that also backs
 // the dashboard's finding-filter.
 function matchesFindingsFilter(row            , filter                )          {
-  return matchesFindingFilterCriteria(row, filter);
+  // A sql-scope finding carries its stages in evidence.stageIds, not a stageId column: it matches
+  // the one stage it touches, as the dashboard's Stage details lists it.
+  const stageIds = 'stageIds' in row.evidence ? row.evidence.stageIds : null;
+  return matchesFindingFilterCriteria({ ...row, stageId: singleStageId({ stageId: row.stageId, stageIds }) }, filter);
 }
 
 /** Build a FindingsFilter from the three optional CLI/MCP filter dimensions, or undefined when
@@ -392,20 +719,21 @@ export function toFindingsFilter(
  * Build a portable evidence report from an appModel.
  * @param opts redact=true pseudonymizes app ids / hosts; markdown=false skips the Markdown string;
  *   findingsFilter narrows json.findings (and the Markdown Findings section) only, summary,
- *   recommendations, and cleanChecks stay computed from the full set, so a narrow filter never
- *   hides that other checks passed or other fixes exist.
+ *   recommendations, cleanChecks and notRunChecks stay computed from the full set, so a narrow filter never
+ *   hides that other checks passed or other fixes exist. thresholds runs the detectors with a user's
+ *   validated overrides (CLI/MCP only) and labels whatever they changed.
  */
 export function buildEvidenceReport(
   appModel          ,
-  { redact = false, markdown: computeMarkdown = true, findingsFilter }   
-                                                                          
+  { redact = false, markdown: computeMarkdown = true, findingsFilter, thresholds }   
+                                                                                                           
     = {},
 )                                                 {
-  let json = buildJson(appModel);
-  if (redact) json = redactReport(json);
+  let json = redact ? buildRedactedJson(appModel, thresholds) : buildJson(appModel, thresholds);
+  const incomplete = json.findings.some((row) => row.type === 'incompleteRun');
   // Filter after redact, not before: redaction only replaces string values on surviving rows,
   // never adds/removes rows, so the two orderings produce identical final content.
   if (findingsFilter) json = { ...json, findings: json.findings.filter((row) => matchesFindingsFilter(row, findingsFilter)) };
-  const markdown = computeMarkdown ? renderMarkdown(json) : '';
+  const markdown = computeMarkdown ? renderMarkdown(json, incomplete) : '';
   return { markdown, json };
 }