sparkforensics-server 0.6.0 → 0.8.1

package/vendor-core/event-schemas.js

@@ -91,19 +91,17 @@ const SparkPlanInfoFieldSchema = z
     }
   });
 
-// Fully-specified event schemas.
-
 // SparkListenerLogStart case (inline: sets state.pendingSparkVersion).
 export const LogStartEventSchema = z.object({
   Event: z.literal('SparkListenerLogStart'),
   'Spark Version': z.string().optional(),
 });
 
-// startApplication.
 export const ApplicationStartEventSchema = z.object({
   Event: z.literal('SparkListenerApplicationStart'),
   'App ID': z.string().optional(),
   'App Name': z.string().optional(),
+  'App Attempt ID': z.string().optional(),
   Timestamp: z.number().optional(),
   'Spark Version': z.string().optional(),
 });
@@ -121,7 +119,6 @@ export const EnvironmentUpdateEventSchema = z.object({
   ]).optional(),
 });
 
-// inline ApplicationEnd case.
 export const ApplicationEndEventSchema = z.object({
   Event: z.literal('SparkListenerApplicationEnd'),
   Timestamp: z.number().optional(),
@@ -150,8 +147,6 @@ export const ExecutorRemovedEventSchema = z.object({
   'Removed Reason': z.string().optional(),
 });
 
-// Schemas transcribed from their handlers.
-
 // startJob. 'spark.sql.execution.id' is read then parseInt'd, which coerces to string internally,
 // so it tolerates the raw value being a number; widened to string|number (some logs emit a JSON number).
 export const JobStartEventSchema = z.object({
@@ -164,7 +159,6 @@ export const JobStartEventSchema = z.object({
   }).optional(),
 });
 
-// endJob.
 export const JobEndEventSchema = z.object({
   Event: z.literal('SparkListenerJobEnd'),
   'Job ID': z.number(),
@@ -230,7 +224,6 @@ export const BlockUpdatedEventSchema = z.object({
   }),
 });
 
-// inline StageCompleted case.
 export const StageCompletedEventSchema = z.object({
   Event: z.literal('SparkListenerStageCompleted'),
   'Stage Info': z.object({
@@ -239,10 +232,17 @@ export const StageCompletedEventSchema = z.object({
     'Submission Time': z.number().optional(),
     'Completion Time': z.number().optional(),
     'Failure Reason': z.string().optional(),
+    // The stage's accumulators, read only for SQL metrics (Metadata "sql"): their Value is the
+    // accumulator's running total, which Spark writes as a decimal string. Value stays unknown
+    // because internal accumulators write arrays and other shapes there.
+    Accumulables: z.array(z.object({
+      ID: z.number(),
+      Value: z.unknown().optional(),
+      Metadata: z.string().optional(),
+    })).optional(),
   }),
 });
 
-// recordStageExecutorMetrics.
 export const StageExecutorMetricsEventSchema = z.object({
   Event: z.literal('SparkListenerStageExecutorMetrics'),
   'Stage ID': z.number(),
@@ -250,9 +250,8 @@ export const StageExecutorMetricsEventSchema = z.object({
   'Executor Metrics': z.record(z.string(), z.number()).optional(),
 });
 
-const MAX_ACCUMULABLES_PER_TASK = 10_000;
+export const MAX_ACCUMULABLES_PER_TASK = 10_000;
 
-// accumulateTask.
 export const TaskEndEventSchema = z.object({
   Event: z.literal('SparkListenerTaskEnd'),
   'Stage ID': z.number(),
@@ -286,12 +285,16 @@ export const TaskEndEventSchema = z.object({
     // Bounded well above any real per-task metric count: caps how much a crafted TaskEnd can grow
     // taskAccumStages, which is never pruned for the life of the parse. Only ID is read. Update and
     // Value stay undeclared: Spark writes some as JSON arrays (internal.metrics.updatedBlockStatuses),
-    // and a string|number check on them rejected the whole TaskEnd, dropping the task from its
-    // stage's stats. They were also ~15% of TaskEnd validation time.
+    // and a string|number check on them would reject the whole TaskEnd, dropping the task from its
+    // stage's stats. Validating them would also add ~15% to TaskEnd validation time.
     Accumulables: z.array(z.object({
       ID: z.number(),
     })).max(MAX_ACCUMULABLES_PER_TASK).optional(),
   }).optional(),
+  // Spark 3.0+: the executor's metric peaks as of this task's end, one number per ExecutorMetricType
+  // name (the same names as SparkListenerStageExecutorMetrics). Values are read by name and kept
+  // only when numeric, so a stray value never drops the task.
+  'Task Executor Metrics': z.record(z.string(), z.unknown()).optional(),
   'Task Metrics': z.object({
     'Peak Execution Memory': z.number().optional(),
     'JVM GC Time': z.number().optional(),
@@ -303,12 +306,14 @@ export const TaskEndEventSchema = z.object({
       'Remote Bytes Read': z.number().optional(),
       'Local Bytes Read': z.number().optional(),
       'Fetch Wait Time': z.number().optional(),
+      'Total Records Read': z.number().optional(),
     }).optional(),
     'Shuffle Write Metrics': z.object({
       'Shuffle Bytes Written': z.number().optional(),
     }).optional(),
     'Input Metrics': z.object({
       'Bytes Read': z.number().optional(),
+      'Records Read': z.number().optional(),
     }).optional(),
     'Output Metrics': z.object({
       'Bytes Written': z.number().optional(),
@@ -317,7 +322,6 @@ export const TaskEndEventSchema = z.object({
   }).optional(),
 });
 
-// startSqlExecution.
 export const SqlExecutionStartEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart'),
   executionId: z.number(),
@@ -326,6 +330,9 @@ export const SqlExecutionStartEventSchema = z.object({
   physicalPlanDescription: z.string().optional(),
   time: z.number(),
   sparkPlanInfo: SparkPlanInfoFieldSchema,
+  // Session settings of this execution that differ from the SparkContext's (Spark 3.3+; redacted
+  // values are masked by Spark). Anything but a string-to-string map is dropped, not a parse error.
+  modifiedConfigs: z.record(z.string(), z.string()).optional().catch(undefined),
 });
 
 // applyAdaptiveExecutionUpdate: AQE re-plans mid-query and re-emits sparkPlanInfo for the same
@@ -337,14 +344,12 @@ export const SqlAdaptiveExecutionUpdateEventSchema = z.object({
   sparkPlanInfo: SparkPlanInfoFieldSchema,
 });
 
-// endSqlExecution.
 export const SqlExecutionEndEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd'),
   executionId: z.number(),
   time: z.number(),
 });
 
-// applyDriverAccumUpdates.
 export const DriverAccumUpdatesEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates'),
   executionId: z.number(),

package/vendor-core/evidence-availability.js

@@ -6,7 +6,7 @@ export const EVIDENCE_AVAILABILITY_SCHEMA_VERSION = 1;
 
 const SUMMARIES                                     = {
   observed: 'Observed in this event log.',
-  explicitlyDisabled: 'Explicitly disabled in this event log.',
+  explicitlyDisabled: 'Disabled in this event log, by its configuration or by Spark\'s default.',
   noObservedExecutorMetrics: 'Not emitted by this event log.',
   noObservedStageSubmission: 'Not emitted by this event log.',
   noRddStorageSnapshot: 'Stages were submitted without RDD storage snapshots.',
@@ -66,6 +66,13 @@ function absent(key             , reasonCode                    , trustworthy
   return trustworthy ? entry(key, reasonCode === 'noSqlExecution' ? 'notApplicable' : 'notEmitted', reasonCode) : entry(key, 'unknown', 'parseIncomplete');
 }
 
+// The run's effective spark.eventLog.logStageExecutorMetrics: the parser's resolution of it (the
+// logged value, else Spark's default of off) when it recorded one, else the logged value alone.
+function stageExecutorMetricsLogging(app                             )          {
+  const resolved = app?.resources?.stageExecutorMetricsLogging;
+  return resolved != null ? resolved : app?.config?.['spark.eventLog.logStageExecutorMetrics'];
+}
+
 export function deriveEvidenceAvailability(appModel          , { skippedLines = 0 }                            = {})                       {
   const app = appModel?.app;
   const inputs                          = app?.evidenceInputs ?? {};
@@ -80,7 +87,7 @@ export function deriveEvidenceAvailability(appModel          , { skippedLines =
 
   const executorMetrics = metricRows > 0
     ? observed('executorMetrics', 'executorMetricRows', metricRows)
-    : configIs(app?.config?.['spark.eventLog.logStageExecutorMetrics'], 'false')
+    : configIs(stageExecutorMetricsLogging(app), 'false')
       ? entry('executorMetrics', 'disabled', 'explicitlyDisabled')
       : absent('executorMetrics', 'noObservedExecutorMetrics', trustworthy);
   const rddStorageSnapshots = snapshots > 0

package/vendor-core/evidence-report.js

@@ -14,15 +14,16 @@ import { findingName, titleCase } from './finding-names.js';
 import { redactReport, redactRunModel } from './redact.js';
 import { formatTaskFailureHeadline,                       } from './task-failure.js';
 import { findingActionLabel } from './finding-action-label.js';
+import { findingKeyer } from './run-comparison.js';
 import { matchesFindingFilterCriteria, singleStageId } from './finding-filter-predicate.js';
 import { buildRecommendationRollup, isEligible, isRealFinding, rankFindings, resourceGroupTotal,                  } from './recommendation-rollup.js';
 import { checkCoverage, isCleanRun } from './check-coverage.js';
-import { buildRunVerdict, stepCopyRecommendation, stepCopyText,                      } from './run-verdict.js';
+import { buildRunVerdict, stepCopyRecommendation, stepCopyText,                                        } from './run-verdict.js';
 import {
-  estimateProvenance, impactEstimateFigure, impactFigure, rawWasteMeaning, savingsMeaning,
+  estimateProvenance, impactEstimateFigure, impactFigure, savingsMeaning,
 } from './impact-format.js';
 import { computeRunShape,               } from './run-shape.js';
-import { extractWriteTargets,                         } from './write-targets.js';
+import { extractWriteTargets, isPlainWrite,                         } from './write-targets.js';
 import { detectorInfoByType } from './detector-docs.js';
              
                                                                                                                      
@@ -41,7 +42,11 @@ export const EVIDENCE_SCHEMA_VERSION         = 5;
 // configAudit's current setting, incompleteRun's 'missing') carry theirs in `valueText` instead,
 // present only on those rows.
                              
-                                                                                        
+                    
+                                                                                              
+                                                                                                    
+                                                                                          
+                                                                                  
                                                                                             
                                                           
                                                                                        
@@ -56,6 +61,9 @@ export const EVIDENCE_SCHEMA_VERSION         = 5;
                                                                                  
                   
                                 
+                                                                                                
+                                      
+                              
                                                                                        
                                                                                    
                                     
@@ -128,12 +136,29 @@ export const EVIDENCE_SCHEMA_VERSION         = 5;
                
  
 
+// One cost-lane entry: a finding ranked by the measured busy core time its fix removes.
+                                     
+                         
+               
+              
+                               
+                      
+                                                                
+                     
+                                                
+                   
+ 
+
 // The dashboard's run verdict (run-verdict.ts): title, summary sentences, the first steps in the
-// same order, how many more places the full list holds, and the "Copy next steps" text.
+// same order, how many more places the full list holds, and the "Copy next steps" text. `steps`
+// rank by wall clock; `costSteps` rank by measured busy core time and list only findings that
+// carry one; `idle` is the run's allocated core time that ran no task.
                               
                 
                     
                           
+                                  
+                           
                           
                           
  
@@ -180,17 +205,17 @@ export const EVIDENCE_SCHEMA_VERSION         = 5;
 // module (stageShape's totalCores, utilization's unrounded fraction) is left off both, so it never
 // reaches the report; adding, renaming or dropping a key here changes the report contract.
 const EVIDENCE_KEYS = {
-  skew: ['origin'],
+  skew: ['origin', 'aqeSkew', 'cause', 'dataRatio'],
   stageShape: ['rule'],
   shuffle: ['partitions'],
-  partitionSizing: ['rule', 'origin', 'partitions'],
+  partitionSizing: ['rule', 'origin', 'aqeSkew', 'partitions'],
   spill: ['spillMagnitude', 'reads'],
   gc: ['direction'],
-  slowHost: ['variant', 'host', 'hostTaskShare', 'hostMeanMs', 'dimension', 'executorId', 'execMaxValue'],
+  slowHost: ['variant', 'host', 'hostTaskShare', 'hostMeanMs', 'dimension', 'executorId', 'execMaxValue', 'executorCount', 'taskCount'],
   stageSlowness: ['reads'],
   stageFailed: ['variant', 'numTasks', 'memoryBytesSpilled', 'failedTaskDetails'],
   failures: ['failedTasks', 'dominantReason', 'dominantError', 'failureGroups', 'otherFailedTasks'],
-  straggler: ['unit', 'speculativeTasks', 'stragglerCount', 'origin'],
+  straggler: ['unit', 'speculativeTasks', 'stragglerCount', 'origin', 'aqeSkew', 'cause', 'causeSharePct', 'host', 'hostTasks', 'cpuPct'],
   speculationWaste: [],
   retryWaste: ['numTasks', 'memoryBytesSpilled', 'retriedTaskDetails'],
   tinyTask: ['reads'],
@@ -211,7 +236,9 @@ const EVIDENCE_KEYS = {
     'groupIndex',
   ],
   smallFiles: ['executionId', 'stageIds', 'fileCount', 'direction', 'nodeName'],
-  underBroadcast: ['executionId', 'stageIds', 'largerSideBytes', 'broadcastThreshold'],
+  pythonUdf: ['executionId', 'stageIds', 'dataSentBytes', 'dataReturnedBytes', 'stageDurationMs'],
+  nestedLoopJoin: ['executionId', 'stageIds', 'nodeName', 'joinType', 'condition', 'outputRows', 'leftRows', 'rightRows'],
+  underBroadcast: ['executionId', 'stageIds', 'largerSideBytes', 'joinType', 'buildSide', 'broadcastThreshold'],
   overBroadcast: ['executionId', 'stageIds', 'broadcastThreshold'],
 }                                                                                    ;
 
@@ -233,11 +260,12 @@ function projectEvidence(f         )                          {
   return evidence;
 }
 
-function findingRow(f         )             {
+function findingRow(f         , key        )             {
   // Cast: projectEvidence's keys come from EVIDENCE_KEYS[f.type], so `evidence` is that type's
   // FindingEvidenceMap entry, which TypeScript can't correlate with `type` on its own.
   const row = {
     id: f.id ?? null,
+    key,
     type: f.type,
     name: titleCase(findingName(f.type)),
     tag: typeTag(f.type),
@@ -262,6 +290,8 @@ function findingRow(f         )             {
     row.impact = figure.text;
     row.impactMeaning = figure.meaning;
   }
+  const provenance = estimateProvenance(f);
+  if (provenance) row.estimateProvenance = provenance;
   if (f.tunedThresholds != null) row.tunedThresholds = f.tunedThresholds;
   return row;
 }
@@ -284,8 +314,8 @@ function sortFindings(rows              )               {
 // file's isEligible call omits FixTheseFirst's REGISTRY check: every finding here already came out
 // of analyze()/auditConfig() so it's a known type, and REGISTRY (.tsx) isn't importable here anyway.
 
-// The impact-ranked "highest-leverage fix" rollup, ported from FixTheseFirst.tsx so CLI/MCP/download
-// get the same ranking. buildRecommendationRollup already returns groups in the correct cross-group
+// The impact-ranked "highest-leverage fix" rollup, so CLI, MCP and download share the dashboard's
+// ranking. buildRecommendationRollup already returns groups in the correct cross-group
 // order, so this only maps each group to its JSON row without re-sorting.
 function buildRecommendations(
   findings           ,
@@ -324,7 +354,7 @@ function buildRecommendations(
         unit: group.unit,
         total: group.total,
         impact: shown,
-        impactMeaning: shown ? rawWasteMeaning(representative.impactEstimate?.rawWaste) : null,
+        impactMeaning: shown ? savingsMeaning(representative) : null,
       };
     }
     return {
@@ -368,6 +398,26 @@ function buildCheckLists(
   return { cleanChecks, notRunChecks };
 }
 
+// A plain write with no output row count (no node metric, and its root execution's stages report no
+// rows written or the root holds another write) is not a check the log could run, so a consumer that
+// compares row counts sees it listed here instead of reading the null as a match.
+const WRITE_FIGURES_CHECK = 'writeOutputRows';
+const LISTED_WRITES = 5;
+
+function notRunWriteFigures(report                    )                          {
+  const missing = report.writes.filter((w) => isPlainWrite(w) && w.outputRows === null);
+  if (missing.length === 0) return null;
+  const named = missing.slice(0, LISTED_WRITES)
+    .map((w) => `execution ${w.sqlExecutionId} ${w.command}`).join(', ');
+  const more = missing.length > LISTED_WRITES ? ` and ${missing.length - LISTED_WRITES} more` : '';
+  return {
+    type: WRITE_FIGURES_CHECK,
+    tag: 'INCMP',
+    thresholdSummary: 'an output row count for every plain write, from its node metric or from the stages of its root execution',
+    reason: `${missing.length} write${missing.length === 1 ? ' has' : 's have'} no output row count (${named}${more}): the write node has no row metric and the stages under its root execution report no rows written, or that root holds more than one write.`,
+  };
+}
+
 function countByImpactBand(findings                               )                   {
   const counts                   = { critical: 0, warning: 0, info: 0 };
   for (const f of findings) if (f.impactBand in counts) counts[f.impactBand                          ] += 1;
@@ -394,6 +444,16 @@ function verdictJson(model                 )              {
         text: stepCopyText(step.lead, recommendation, step.stageId),
       };
     }),
+    costSteps: model.costLane.map(({ finding, coreTimeMs, figure })                     => ({
+      stageId: singleStageId(finding),
+      type: finding.type,
+      tag: typeTag(finding.type),
+      leadFindingId: finding.id ?? null,
+      actionLabel: findingActionLabel(finding),
+      coreTimeMs,
+      coreTime: figure,
+    })),
+    idle: model.idle,
     remainingPlaces: model.remaining,
     copyText: model.copyText,
   };
@@ -401,17 +461,17 @@ function verdictJson(model                 )              {
 
 // Keyed by appModel object identity: mcp-tools.ts caches one fixed appModel per runId (never
 // mutated), so re-running analyze()/auditConfig() reproduces the same catalog. getFindingEvidence
-// calls buildEvidenceReport once per drill-down; without this, N lookups meant N detector re-runs.
+// calls buildEvidenceReport once per drill-down; the cache spares each lookup a detector re-run.
 // A WeakMap needs no invalidation: once mcp-tools.ts evicts the appModel, this entry is collectible.
 // Each cache is split first by the overrides object the report ran under (one fixed, frozen object
 // per CLI invocation or MCP server process; DEFAULT_THRESHOLDS for the specification's).
-                                                                                 
+                                                                                        
 const DEFAULT_THRESHOLDS = {};
 const jsonCache              = new WeakMap();
 // The redacted report, keyed by the unredacted appModel it was built from.
 const redactedJsonCache              = new WeakMap();
 
-function cacheFor   (cache                , thresholds                                )                       {
+export function cacheFor   (cache                , thresholds                                )                       {
   const key = thresholds ?? DEFAULT_THRESHOLDS;
   let byModel = cache.get(key);
   if (!byModel) {
@@ -435,7 +495,7 @@ export function runFindings(appModel          , thresholds
     jobs ?? new Map(), sql ?? new Map(),
     runAggregates ?? null, { thresholds },
   );
-  const result = { catalog, config: auditConfig(app) };
+  const result = { catalog, config: auditConfig(app, stages) };
   cache.set(appModel, result);
   return result;
 }
@@ -449,8 +509,8 @@ function buildRedactedJson(appModel          , thresholds
   if (cached) return cached;
   const { catalog, config } = runFindings(appModel, thresholds);
   const run = redactRunModel(appModel, catalog, config);
-  // redactReport stays as a last pass: idempotent over pseudonyms, and it covers the report's own
-  // structured fields (summary.app.id) the same way it always has.
+  // redactReport runs as a last pass: idempotent over pseudonyms, and it covers the report's own
+  // structured fields (summary.app.id).
   const result = redactReport(buildJson(run.appModel, thresholds, { catalog: run.catalog, config: run.configFindings }));
   cache.set(appModel, result);
   return result;
@@ -465,9 +525,15 @@ function buildJson(
   const { app, stages, executors, sql, jobs, evidenceAvailability } = appModel;
   const { catalog, config } = findings ?? runFindings(appModel, thresholds);
   const allFindings = [...catalog, ...config];
-  const rows = sortFindings(allFindings.map(findingRow));
+  const keyOf = findingKeyer({ stages: stages ?? new Map(), sql: sql ?? new Map() });
+  const rows = sortFindings(allFindings.map((f) => findingRow(f, keyOf(f))));
   const recommendations = buildRecommendations(allFindings, stages ?? new Map());
   const { cleanChecks, notRunChecks } = buildCheckLists(allFindings, stages ?? new Map(), thresholds);
+  const writeTargets = extractWriteTargets(sql ?? new Map(), {
+    stages, skippedLines: appModel.skippedLines, unreadableSqlExecutions: appModel.unreadableSqlExecutions,
+  });
+  const unfiguredWrites = notRunWriteFigures(writeTargets);
+  if (unfiguredWrites) notRunChecks.push(unfiguredWrites);
   const tuned = tunedDetectors(thresholds);
   const actionable = allFindings.filter(isEligible);
   const fullModel           = {
@@ -480,8 +546,6 @@ function buildJson(
     schemaVersion: EVIDENCE_SCHEMA_VERSION,
     summary: {
       app: {
-        // `?? null`, not `?? undefined`: JSON.stringify drops undefined keys but keeps null, and
-        // sparkVersion's null is a deliberate "unknown/absent" sentinel; undefined would drop these.
         id: app?.id ?? null,
         name: app?.name ?? null,
         sparkVersion: app?.sparkVersion ?? null,
@@ -509,9 +573,7 @@ function buildJson(
     // Order follows DETECTORS (stable) => byte-stable serialization.
     detectors: tunedDetectorCatalog(thresholds),
     findings: rows,
-    writeTargets: extractWriteTargets(sql ?? new Map(), {
-      skippedLines: appModel.skippedLines, unreadableSqlExecutions: appModel.unreadableSqlExecutions,
-    }),
+    writeTargets,
     recommendations,
     cleanChecks,
     notRunChecks,
@@ -592,8 +654,9 @@ function renderRunShape(shape          )           {
 function renderVerdict(verdict             )           {
   const lines = ['## Verdict', '', verdict.title];
   if (verdict.summary.length > 0) lines.push('', verdict.summary.join(' '));
+  if (verdict.idle) lines.push('', verdict.idle.text);
   if (verdict.steps.length > 0) {
-    lines.push('');
+    lines.push('', 'By wall clock (run time):', '');
     verdict.steps.forEach((step, i) => {
       lines.push(`${i + 1}. [${step.tag}] ${step.text}`);
       if (step.relatedTypes.length > 0) {
@@ -606,6 +669,13 @@ function renderVerdict(verdict             )           {
       lines.push('', `${places} to look at in the Findings section below.`);
     }
   }
+  if (verdict.costSteps.length > 0) {
+    lines.push('', 'By core time (measured busy core time a fix removes):', '');
+    verdict.costSteps.forEach((step, i) => {
+      const where = step.stageId != null ? ` in Stage ${step.stageId}` : '';
+      lines.push(`${i + 1}. [${step.tag}] ${step.actionLabel}${where}: ${step.coreTime} of core time.`);
+    });
+  }
   lines.push('');
   return lines;
 }
@@ -713,6 +783,10 @@ function renderMarkdown(json                    , incomplete         )         {
                         
                   
                    
+                                                         
+                       
+                                                                                  
+                       
  
 
 // CLI/MCP-facing filter over FindingRow, delegating to the shared core predicate that also backs
@@ -721,21 +795,24 @@ function matchesFindingsFilter(row            , filter                )
   // A sql-scope finding carries its stages in evidence.stageIds, not a stageId column: it matches
   // the one stage it touches, as the dashboard's Stage details lists it.
   const stageIds = 'stageIds' in row.evidence ? row.evidence.stageIds : null;
+  if (filter.minBand && (IMPACT_BAND_ORDER[row.impactBand] ?? 9) > IMPACT_BAND_ORDER[filter.minBand]) return false;
   return matchesFindingFilterCriteria({ ...row, stageId: singleStageId({ stageId: row.stageId, stageIds }) }, filter);
 }
 
-/** Build a FindingsFilter from the three optional CLI/MCP filter dimensions, or undefined when
- * none were passed (the "is anything set" gate before calling buildEvidenceReport). */
+/** Build a FindingsFilter from the optional CLI/MCP filter dimensions, or undefined when none were
+ * passed (the "is anything set" gate before calling buildEvidenceReport). */
 export function toFindingsFilter(
-  impactBand           , type           , stageId         ,
+  impactBand           , type           , stageId         , minBand             , maxFindings         ,
 )                             {
-  return (impactBand || type || stageId !== undefined) ? { impactBand, type, stageId } : undefined;
+  return (impactBand || type || stageId !== undefined || minBand || maxFindings !== undefined)
+    ? { impactBand, type, stageId, minBand, maxFindings }
+    : undefined;
 }
 
 /**
  * Build a portable evidence report from an appModel.
  * @param opts redact=true pseudonymizes app ids / hosts; markdown=false skips the Markdown string;
- *   findingsFilter narrows json.findings (and the Markdown Findings section) only, summary,
+ *   findingsFilter (including its minBand and maxFindings limits) narrows json.findings (and the Markdown Findings section) only, summary,
  *   recommendations, cleanChecks and notRunChecks stay computed from the full set, so a narrow filter never
  *   hides that other checks passed or other fixes exist. thresholds runs the detectors with a user's
  *   validated overrides (CLI/MCP only) and labels whatever they changed.
@@ -745,12 +822,18 @@ export function buildEvidenceReport(
   { redact = false, markdown: computeMarkdown = true, findingsFilter, thresholds }   
                                                                                                            
     = {},
-)                                                 {
+)                                                                          {
   let json = redact ? buildRedactedJson(appModel, thresholds) : buildJson(appModel, thresholds);
   const incomplete = json.findings.some((row) => row.type === 'incompleteRun');
   // Filter after redact, not before: redaction only replaces string values on surviving rows,
   // never adds/removes rows, so the two orderings produce identical final content.
-  if (findingsFilter) json = { ...json, findings: json.findings.filter((row) => matchesFindingsFilter(row, findingsFilter)) };
+  // `findingsMatched`: the rows that passed every filter but `maxFindings`, so a caller can say how many that limit left out.
+  let findingsMatched = json.findings.length;
+  if (findingsFilter) {
+    const kept = json.findings.filter((row) => matchesFindingsFilter(row, findingsFilter));
+    findingsMatched = kept.length;
+    json = { ...json, findings: findingsFilter.maxFindings !== undefined ? kept.slice(0, findingsFilter.maxFindings) : kept };
+  }
   const markdown = computeMarkdown ? renderMarkdown(json, incomplete) : '';
-  return { markdown, json };
+  return { markdown, json, findingsMatched };
 }

package/vendor-core/executor-peaks.js

@@ -0,0 +1,41 @@
+// Per-executor JVM heap peaks from the two places a Spark event log can carry them: the executor
+// metrics every TaskEnd reports (Spark 3.0+, always logged) and SparkListenerStageExecutorMetrics
+// rows (only with spark.eventLog.logStageExecutorMetrics=true). Spark's own AppStatusListener
+// folds the TaskEnd values into executor peak memory
+// (https://github.com/apache/spark/blob/v4.0.0/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala).
+
+/** The id Spark gives the driver's own metrics row; the driver's heap is sized by spark.driver.memory. */
+export const DRIVER_EXECUTOR_ID = 'driver';
+
+                       
+                                                                                               
+                                                                                          
+ 
+
+// One analyze() context is asked once per stage detector call: fold it once.
+const cache = new WeakMap                             ();
+
+/** Executor id to its largest sampled JVMHeapMemory in bytes. Spark polls executor metrics at
+ * heartbeat (spark.executor.metrics.pollingInterval defaults to 0), so a peak is a lower bound.
+ * JVMHeapMemory is `MemoryMXBean.getHeapMemoryUsage().getUsed()`, which counts uncollected garbage.
+ * An executor whose every sample is zero (local mode reports zeros) is left out: no measurement. */
+export function executorHeapPeaks(source             )                      {
+  const known = cache.get(source);
+  if (known) return known;
+  const peaks = new Map                ();
+  const take = (executorId        , heap         )       => {
+    // The driver's heap is sized by spark.driver.memory, not the executor setting.
+    if (executorId === DRIVER_EXECUTOR_ID) return;
+    if (typeof heap === 'number' && heap > (peaks.get(executorId) ?? 0)) peaks.set(executorId, heap);
+  };
+  for (const [executorId, metrics] of Object.entries(source.runAggregates?.executorPeakMetrics ?? {})) {
+    take(executorId, metrics?.jvmHeapMemory);
+  }
+  for (const stage of source.stages.values()) {
+    const rows = stage.executorMetrics;
+    if (!(rows instanceof Map)) continue;
+    for (const [executorId, metrics] of rows) take(executorId, metrics?.jvmHeapMemory);
+  }
+  cache.set(source, peaks);
+  return peaks;
+}

package/vendor-core/export-data.js

@@ -4,11 +4,11 @@
                     
                                                                  
 
-/** Version 2 carries the interpretation layer (`interpretation`) and `provenance`. Version 3 types
- * findings per detector: a text figure (stageFailed, configAudit, incompleteRun) moved from `value`
- * to `valueText`, and Plan Advisor findings always carry `planNodeIds`. The export bundle renders
- * only the version it was built for and refuses any other, since an older payload lacks the
- * conclusions it would otherwise have to derive, or carries them in fields this bundle no longer reads. */
+/** The export payload format. It carries the interpretation layer (`interpretation`) and `provenance`,
+ * and types findings per detector: a text figure (stageFailed, configAudit, incompleteRun) is in
+ * `valueText`, and Plan Advisor findings always carry `planNodeIds`. The export bundle renders only
+ * this version and refuses any other: a payload of another version lacks the conclusions the bundle
+ * would otherwise have to derive, or carries them in fields it does not read. */
 export const EXPORT_DATA_SCHEMA_VERSION = 3;
 
 /** @sparkforensics/core's own version (packages/core/package.json, which a test keeps in sync).