sparkforensics-server 0.2.4 → 0.4.0

package/public/index.html

@@ -23,13 +23,14 @@
           document.documentElement.setAttribute('data-theme', 'light');
       } catch (_) {}
     </script>
-    <script type="module" crossorigin src="./assets/index-pnImGbgQ.js"></script>
+    <script type="module" crossorigin src="./assets/index-CEM9bz_x.js"></script>
     <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
     <link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
     <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-B0sEms9Z.js">
-    <link rel="modulepreload" crossorigin href="./assets/react-vendor-D2TLlSzI.js">
+    <link rel="modulepreload" crossorigin href="./assets/react-vendor-BAKt3vny.js">
+    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-LYzHroHO.js">
     <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-DLioOiRN.css">
-    <link rel="stylesheet" crossorigin href="./assets/index-CjVklAdW.css">
+    <link rel="stylesheet" crossorigin href="./assets/index-BcqTLKFp.css">
   </head>
   <body>
     <div id="root"></div>

package/vendor-core/analyzer.js

@@ -1,14 +1,28 @@
-import { DETECTORS,               } from './detectors.js';
+import { DETECTORS, ENTRY_BY_TYPE,                                                                                     } from './detectors.js';
+import { effectiveThresholds, findingTunedThresholds, overridesFor, tunedThresholdsNote } from './threshold-overrides.js';
 import { computePeakConcurrentCores } from './core-count.js';
 import { assertNever } from './assert-never.js';
-import { estimateImpact } from './impact-estimator.js';
+import { estimateImpact,                  } from './impact-estimator.js';
 import { computeOccupancy,                     } from './occupancy.js';
-import { deriveImpactBand } from './impact-band.js';
+import { deriveImpactBand, IMPACT_FLOOR_PCT_CRIT, IMPACT_FLOOR_PCT_WARN,                       } from './impact-band.js';
 import { IMPACT_BAND_ORDER } from './format-utils.js';
              
-                                                                                
+                                                                                                                         
                     
 
+// The runner reads each entry through the Detector contract, not its own precise `as const` shape:
+// the scope switch below calls each entry's bound detect with that scope's target.
+const detectors                      = DETECTORS;
+
+                                 
+                                                                                              
+                                                                                                  
+                                                                                                
+                                                                                              
+                                                                    
+                                  
+ 
+
 // FNV-1a 32-bit stable string hash: deterministic finding id across runs, no timestamps/randomness.
 function fnv1a(str        )         {
   let h = 0x811c9dc5;
@@ -19,28 +33,59 @@ function fnv1a(str        )         {
   return (h >>> 0).toString(16).padStart(8, '0');
 }
 
+// The id hash's discriminator slots, in the order it has always joined them: reordering or renaming
+// one changes every finding id, which needs an EVIDENCE_SCHEMA_VERSION bump.
+const DISCRIMINATOR_SLOTS = [
+  'host', 'executorId', 'rule', 'variant', 'dimension',
+  'direction', 'nodeName', 'rootName', 'subtreeSize', 'groupIndex', 'largerSideBytes',
+  'rddId', 'relation', 'format', 'operator', 'executionIds',
+]         ;
+                                                              
+
+// Per type, the fields that tell apart sibling findings sharing one location and metric; without
+// them those siblings hash to one id:
+//   slowHost (host), memoryUtilization (executorId), partitionSizing (rule);
+//   cacheUtilization (rddId+variant), cachingOpportunity (relation/format for leaf,
+//     operator+relation for composite, executionIds as last resort);
+//   smallFiles (direction/nodeName), duplicatePlanSubtree (groupIndex is the real
+//     uniqueness guarantee: rootName+subtreeSize can collide across groups),
+//     underBroadcast (value+largerSideBytes per node/side).
+// memoryUtilization leaves out `rule`: one heap band per executor, so executorId is already
+// unique and folding rule in risks id churn if band logic changes. partitionSizing keeps
+// `rule`: a stage can emit several rules at once sharing stageId+metric.
+const ID_DISCRIMINATORS                                                                              = {
+  skew: [], stageShape: ['rule'], shuffle: [], partitionSizing: ['rule'], spill: [], gc: ['direction'],
+  slowHost: ['host', 'executorId', 'variant', 'dimension'], stageSlowness: [], stageFailed: ['variant'],
+  failures: [], straggler: [], speculationWaste: [], retryWaste: [], tinyTask: [],
+  incompleteRun: [], coldStart: [], utilization: [], memoryUtilization: ['executorId', 'variant'],
+  cacheUtilization: ['variant', 'rddId'], coreLocality: [], autoscalingChurn: [],
+  cachingOpportunity: ['variant', 'relation', 'format', 'operator', 'executionIds'],
+  jobFailureRate: [], configAudit: [],
+  duplicatePlanSubtree: ['rootName', 'subtreeSize', 'groupIndex'], smallFiles: ['direction', 'nodeName'],
+  underBroadcast: ['largerSideBytes'], overBroadcast: [],
+};
+
+// ID_DISCRIMINATORS as Sets, built once: findingId runs once per finding.
+const ID_DISCRIMINATOR_SETS = new Map                                        (
+  Object.entries(ID_DISCRIMINATORS).map(([type, slots]) => [type, new Set                   (slots)]),
+);
+
+// Location key: stage, else SQL execution, else audited config property.
+function locationKey(f         )                  {
+  if (f.stageId != null) return f.stageId;
+  if ('executionId' in f) return f.executionId;
+  if (f.type === 'configAudit') return f.property;
+  return '';
+}
+
 export function findingId(f         )         {
-  // Location key: stage, else SQL execution, else audited config property.
-  const locKey = f.stageId ?? f.executionId ?? f.property ?? '';
-  // Discriminators for detectors that emit multiple findings on the same
-  // location+metric; without them these siblings hash to one id:
-  //   slowHost (host), memoryUtilization (executorId+dimension), partitionSizing (rule);
-  //   cacheUtilization (rddId+variant), cachingOpportunity (relation/format for leaf,
-  //     operator+relation for composite, executionIds as last resort);
-  //   smallFiles (direction/nodeName), duplicatePlanSubtree (groupIndex is the real
-  //     uniqueness guarantee: rootName+subtreeSize can collide across groups),
-  //     broadcastSizing (value+largerSideBytes per node/side).
-  // memoryUtilization excludes `rule`: one heap band per executor, so executorId is already
-  // unique and folding rule in risks id churn if band logic changes. partitionSizing keeps
-  // `rule`: a stage can emit several rules at once sharing stageId+metric.
-  const rule = f.type === 'memoryUtilization' ? undefined : f.rule;
-  const disc = [
-    f.host, f.executorId, rule, f.variant, f.dimension,
-    f.direction, f.nodeName, f.rootName, f.subtreeSize, f.groupIndex, f.largerSideBytes,
-    f.rddId, f.relation, f.format, f.operator,
-    f.executionIds ? f.executionIds.join(',') : '',
-  ].map((v) => v ?? '').join('|');
-  return fnv1a(`${f.type}|${locKey}|${f.metric ?? ''}|${f.value ?? ''}|${disc}`);
+  const fields = f                                                          ;
+  const listed = ID_DISCRIMINATOR_SETS.get(f.type);
+  const disc = DISCRIMINATOR_SLOTS.map((slot) => {
+    const v = listed?.has(slot) ? fields[slot] : undefined;
+    return Array.isArray(v) ? v.join(',') : v ?? '';
+  }).join('|');
+  return fnv1a(`${f.type}|${locationKey(f)}|${f.metric ?? ''}|${f.value ?? f.valueText ?? ''}|${disc}`);
 }
 
 // skew's max/median branch (stage.taskCount below minTasksForP95) and straggler are both driven
@@ -72,13 +117,13 @@ function flagSkewStragglerOverlap(findings           )       {
   }
 }
 
-function push(out           , entry          , result                            )       {
+function push(out           , entry          , result                            , tuned                         = null)       {
   if (!result) return;
   const detectorVersion = entry.version ?? 1;
   for (const f of (Array.isArray(result) ? result : [result])) {
     if (!f) continue;
-    if (entry.suppressWhen && entry.suppressWhen(f, out)) continue;
     const stamped = { ...f, docAnchor: f.docAnchor ?? entry.docAnchor, detectorVersion };
+    if (tuned) stamped.tunedThresholds = tuned;
     const id = findingId(stamped);
     // Dedup guard: same id => same finding, keep first. Correctness depends on
     // findingId's discriminators being unique per distinct finding, not on this line.
@@ -87,6 +132,50 @@ function push(out           , entry          , result
   }
 }
 
+// A tuned finding's caveat, once its estimate is known: only a finding with an estimate figure
+// (wall-clock or raw waste) says that figure is unvalidated.
+function noteTunedThresholds(findings           )       {
+  for (const f of findings) {
+    if (!f.tunedThresholds) continue;
+    const hasFigure = f.impactEstimate != null && f.impactEstimate.basis !== 'informational';
+    const note = tunedThresholdsNote(f.tunedThresholds, hasFigure);
+    f.validationRequired = f.validationRequired ? `${f.validationRequired} ${note}` : note;
+  }
+}
+
+// skew and straggler gate on floorPctWarn/floorPctCrit thresholds that default to the band's own
+// floors, so a finding they admit at their warn floor grades at least warning. A tuned floor grades
+// that entry's findings too; a type whose entry has no such threshold keeps the defaults.
+function bandFloors(type        , overrides                    )                          {
+  const entry = ENTRY_BY_TYPE.get(type);
+  if (!entry) return null;
+  const { floorPctWarn, floorPctCrit } = effectiveThresholds(entry, overrides);
+  return {
+    warnPct: typeof floorPctWarn === 'number' ? floorPctWarn : IMPACT_FLOOR_PCT_WARN,
+    critPct: typeof floorPctCrit === 'number' ? floorPctCrit : IMPACT_FLOOR_PCT_CRIT,
+  };
+}
+
+// An entry's `suppressedBy` names another entry: drop its findings on every stage that entry
+// flagged. Runs once every detector has, so neither declaration order matters, and reads the
+// unsuppressed findings, so the result doesn't depend on which suppression is applied first.
+function applySuppression(out           )            {
+  const dropped = new Map                     ();
+  for (const entry of detectors) {
+    if (!entry.suppressedBy) continue;
+    const suppressorTypes = new Set        (
+      detectors.filter((d) => d.type === entry.suppressedBy).flatMap((d) => d.emits),
+    );
+    for (const type of entry.emits) {
+      const stagesToDrop = dropped.get(type) ?? new Set        ();
+      for (const f of out) if (suppressorTypes.has(f.type) && f.stageId != null) stagesToDrop.add(f.stageId);
+      dropped.set(type, stagesToDrop);
+    }
+  }
+  if (dropped.size === 0) return out;
+  return out.filter((f) => f.stageId == null || !dropped.get(f.type)?.has(f.stageId));
+}
+
 // `app` widened to `SparkAppInfo | null` to match real callers (AppModel.app is
 // nullable at the type level); every detector below tolerates a null app.
 export function analyze(
@@ -97,6 +186,7 @@ export function analyze(
   jobs                  ,
   sql                            = new Map(),
   runAggregates                       = null,
+  { thresholds }                 = {},
 )            {
   // `app ?? {}`: detectors tolerate a null app (malformed logs), so this must too.
   // computePeakConcurrentCores (not computeTotalCores): the occupancy ceiling needs a
@@ -108,37 +198,55 @@ export function analyze(
     executorsAdded                                                                         ,
     executorsRemoved                                                    ,
   );
-  // Computed once so detectors gate impact band on the same occupancy-clipped waste
-  // estimateImpact displays as savings, not a raw pre-clip delta the two passes would disagree on.
-  const occupancy = computeOccupancy(stages                                          , totalCores);
-  const ctx = {
-    app, stages, executorsAdded, executorsRemoved, jobs, sql, runAggregates, occupancy,
+  // One occupancy sweep per analysis, shared by the detectors' runtime floors and every entry's
+  // estimate(), so a floor gates on the same occupancy-clipped figure displayed as savings.
+  const impact              = {
+    stages, totalCores,
+    occupancy: computeOccupancy(stages                                          , totalCores),
+  };
+  // The one cast from the posted-model types to the detector-side shapes: types.ts's Stage and
+  // SqlExecution carry a catch-all index signature, while every field DetectorStage/DetectorSqlExec
+  // declare is one finalizeStage and event-handlers.ts always set (see detectors.ts's header).
+  const ctx              = {
+    app, jobs, executorsAdded, executorsRemoved, runAggregates, impact,
+    stages: stages                                    ,
+    sql: sql                                 ,
   };
   const out            = [];
-  for (const d of DETECTORS) {
+  for (const d of detectors) {
     if (d.inScorecard === false) continue;
+    const overrides = overridesFor(d, thresholds);
+    const tuned = findingTunedThresholds(d, thresholds);
     switch (d.scope) {
-      case 'stage':
-        for (const s of stages.values()) push(out, d, d.detect(s, ctx));
+      case 'stage': {
+        const detect = d.withThresholds(overrides);
+        for (const s of ctx.stages.values()) push(out, d, detect(s, ctx), tuned);
         break;
-      case 'sql':
-        for (const e of sql.values()) push(out, d, d.detect(e, ctx));
+      }
+      case 'sql': {
+        const detect = d.withThresholds(overrides);
+        for (const e of ctx.sql.values()) push(out, d, detect(e, ctx), tuned);
         break;
+      }
       case 'app':
+        push(out, d, d.withThresholds(overrides)(ctx), tuned);
+        break;
       case 'config':
-        push(out, d, d.detect(ctx));
+        push(out, d, d.withThresholds(overrides)(ctx                               ), tuned);
         break;
       default:
-        assertNever(d.scope);
+        assertNever(d);
     }
   }
-  estimateImpact(out, stages, totalCores);
-  deriveImpactBand(out, app);
-  flagSkewStragglerOverlap(out);
+  const findings = applySuppression(out);
+  estimateImpact(findings, impact);
+  noteTunedThresholds(findings);
+  deriveImpactBand(findings, app, thresholds ? (type) => bandFloors(type, thresholds) : undefined);
+  flagSkewStragglerOverlap(findings);
   // Ascending IMPACT_BAND_ORDER (critical 0 -> info 2) puts the worst band first;
   // stable sort keeps DETECTORS declaration order within a band.
-  out.sort((a, b) => IMPACT_BAND_ORDER[a.impactBand] - IMPACT_BAND_ORDER[b.impactBand]);
-  return out;
+  findings.sort((a, b) => IMPACT_BAND_ORDER[a.impactBand] - IMPACT_BAND_ORDER[b.impactBand]);
+  return findings;
 }
 
 // Memoizes auditConfig by `app` identity (like evidence-report.ts's jsonCache) so config detectors
@@ -149,10 +257,10 @@ const auditConfigCache = new WeakMap                         ();
 
 function computeAuditConfig(app                     )            {
   const out            = [];
-  for (const d of DETECTORS) if (d.scope === 'config') push(out, d, d.detect({ app }));
-  // configAudit's impact case is unconditionally costOnly('none'): needs no stages/totalCores,
-  // an empty stages map gives parity with analyze().
-  estimateImpact(out, new Map());
+  for (const d of detectors) if (d.scope === 'config') push(out, d, d.withThresholds()({ app }));
+  // configAudit's estimate is unconditionally costOnly('none'): needs no stages/totalCores, so an
+  // empty context gives parity with analyze().
+  estimateImpact(out, { stages: new Map(), occupancy: new Map(), totalCores: 0 });
   deriveImpactBand(out, app);
   return out.map((f) => ({ ...f, stageId: f.stageId ?? null }));
 }

package/vendor-core/check-coverage.js

@@ -0,0 +1,88 @@
+// Which checks a log could actually run. Shared by the dashboard (verdict, top bar, Clean checks)
+// and the CLI/MCP evidence report, so a check the log lacked the data for never reads as passed on
+// one path and "not checked" on another.
+import { detectorCatalog } from './detectors.js';
+import { isRealFinding } from './recommendation-rollup.js';
+import { summarizeRunOutcome } from './run-outcome.js';
+                                                    
+
+export const NO_FINISHED_STAGE_GAP = 'No stage in this log recorded an end, so the stage checks had nothing to measure.';
+export const INCOMPLETE_RUN_GAP =
+  'The log has no end-of-run record, so the core usage, memory and executor churn checks had no run length to measure.';
+
+/** True when at least one stage recorded both a start and an end, so the
+ * stage checks had something to measure. */
+export function hasFinishedStage(stages                    )          {
+  return [...stages.values()].some((stage) => stage.submittedAt != null && stage.completedAt != null);
+}
+
+/** A finding that reports a check could not run for lack of evidence (cache
+ * storage without block updates, memory without executor metrics), rather
+ * than a problem found. Its recommendation names the setting to turn on. */
+export function isEvidenceCaveat(finding         )          {
+  return ('dataUnavailable' in finding && finding.dataUnavailable === true) || !isRealFinding(finding);
+}
+
+/** App-level checks measured over the run's full span, which a log with no
+ * ApplicationEnd (an `incompleteRun` finding) cannot give them. */
+export const RUN_SPAN_CHECK_TYPES                      = new Set(['utilization', 'memoryUtilization', 'autoscalingChurn']);
+
+/** Detector types that measure each stage, read from the detector catalog. */
+export const PER_STAGE_CHECK_TYPES                      = new Set(
+  detectorCatalog().filter((entry) => entry.scope === 'stage').map((entry) => entry.type),
+);
+
+export function isIncompleteRun(allFindings           )          {
+  return allFindings.some((finding) => finding.type === 'incompleteRun');
+}
+
+/** What this log could not check, in plain sentences, each saying what to
+ * turn on for the next run where the detector names it. */
+export function verdictGaps(allFindings           , noFinishedStages         )           {
+  const gaps = new Set        ();
+  if (noFinishedStages) gaps.add(NO_FINISHED_STAGE_GAP);
+  if (isIncompleteRun(allFindings)) gaps.add(INCOMPLETE_RUN_GAP);
+  for (const finding of allFindings) {
+    if (isEvidenceCaveat(finding) && finding.recommendation) gaps.add(finding.recommendation);
+  }
+  return [...gaps];
+}
+
+/** The one rule for calling a run clean, shared by the verdict, the top bar
+ * and the evidence report: no finding at all, no failed job, and nothing the
+ * log lacked to run a check. */
+export function isCleanRun(appModel                                   , allFindings           )          {
+  if (summarizeRunOutcome(appModel.jobs, allFindings).failedJobs > 0) return false;
+  if (allFindings.some(isRealFinding)) return false;
+  return verdictGaps(allFindings, !hasFinishedStage(appModel.stages)).length === 0;
+}
+
+                                
+                                                                                                  
+                                  
+                                                                                        
+                                                                                        
+                                            
+ 
+
+/** The not-run rule for one run: an evidence caveat of that type, a per-stage
+ * check on a log where no stage finished, or a run-span check on a log with
+ * no ApplicationEnd. */
+export function checkCoverage(stages                    , allFindings           )                {
+  const noFinishedStages = !hasFinishedStage(stages);
+  const incomplete = isIncompleteRun(allFindings);
+  const caveatReasons = new Map                       ();
+  for (const finding of allFindings) {
+    if (!isEvidenceCaveat(finding)) continue;
+    if (!caveatReasons.get(finding.type)) caveatReasons.set(finding.type, finding.recommendation ?? null);
+  }
+  const notRunReason = (type        )                => {
+    const caveatReason = caveatReasons.get(type);
+    if (caveatReason) return caveatReason;
+    if (noFinishedStages && PER_STAGE_CHECK_TYPES.has(type)) return NO_FINISHED_STAGE_GAP;
+    if (incomplete && RUN_SPAN_CHECK_TYPES.has(type)) return INCOMPLETE_RUN_GAP;
+    // A caveat with no recommendation still means the check did not run.
+    return caveatReasons.has(type) ? 'The log lacked the data this check needs.' : null;
+  };
+  return { isNotRun: (type) => notRunReason(type) != null, notRunReason };
+}

package/vendor-core/cli/budgets.js

@@ -1,5 +1,6 @@
 import { computeEfficiencyModel } from '../efficiency-model.js';
-import { computeSkewRatio, DETECTORS } from '../detectors.js';
+import { computeSkewRatio, ENTRY_BY_TYPE,                         } from '../detectors.js';
+import { effectiveThresholds } from '../threshold-overrides.js';
 import { IMPACT_BAND_ORDER } from '../format-utils.js';
                                                               
                                                                  
@@ -13,26 +14,26 @@ const IMPACT_BANDS = Object.keys(IMPACT_BAND_ORDER)                ;
  
                                
                                                                                             
-                                              
+                                                               
                                                 
                  
  
 
-const skewDetector = DETECTORS.find((d) => d.type === 'skew');
-const SKEW_MIN_TASKS_FOR_P95 = skewDetector .thresholds .minTasksForP95          ;
+// The skew entry's minTasksForP95 under the run's overrides, so the budget measures the same
+// ratio (P95/median or max/median) the skew finding reports.
+function skewMinTasksForP95(thresholds                                )         {
+  return effectiveThresholds(ENTRY_BY_TYPE.get('skew') , thresholds).minTasksForP95          ;
+}
 
 function taskDataTrusted(appModel          )          {
   const entry = appModel.evidenceAvailability?.entries?.find((e) => e.key === 'taskCoreTime');
   return entry?.state === 'present';
 }
 
-// Finding.value is number|string (some detectors put text there); spill findings are
-// always numeric, so the typeof guard narrows without changing behavior for real input.
 function maxFindingValue(catalog           , type        )                {
   const values = catalog
     .filter((f) => f.type === type)
-    .map((f) => f.value ?? 0)
-    .filter((v)              => typeof v === 'number');
+    .map((f) => f.value ?? 0);
   return values.length > 0 ? Math.max(...values) : null;
 }
 
@@ -58,7 +59,7 @@ function checkSpill(appModel          , catalog           , maxSpillGb        )
     : { name: 'max-spill', status: 'pass', detail: `Peak stage spill ${maxBytes} bytes within budget ${budgetBytes} bytes.` };
 }
 
-function checkSkew(appModel          , maxSkewRatio        )               {
+function checkSkew(appModel          , maxSkewRatio        , minTasksForP95        )               {
   if (!taskDataTrusted(appModel)) {
     return { name: 'max-skew', status: 'inconclusive', detail: 'No trustworthy task-level evidence to measure skew.' };
   }
@@ -69,7 +70,7 @@ function checkSkew(appModel          , maxSkewRatio        )               {
   // Recompute the true ratio per stage: the skew detector floors findings at
   // thresholds.ratioWarn (3), so a stricter budget can't be enforced from catalog alone.
   const ratios = stages
-    .map((stage) => computeSkewRatio(stage, SKEW_MIN_TASKS_FOR_P95))
+    .map((stage) => computeSkewRatio(stage, minTasksForP95))
     .filter((r) => r !== null)
     .map((r) => r.ratio);
   if (ratios.length === 0) {
@@ -97,21 +98,24 @@ function checkFailedTaskRate(appModel          , catalog           , maxPct
   return { name: 'max-failed-task-rate', status: 'pass', detail: `No job-failure-rate finding: task failure rate is below the detector's reporting floor.` };
 }
 
+// Busy core time: the share of available executor core time that ran tasks, 100 minus the
+// dashboard's "Unused core time". Not the dashboard's Efficiency tile (the share of wall-clock with
+// a stage running), so the detail never calls it "Efficiency".
 function checkEfficiency(appModel          , minPct        )               {
   if (!taskDataTrusted(appModel)) {
-    return { name: 'min-efficiency', status: 'inconclusive', detail: 'No trustworthy task-level evidence to measure efficiency.' };
+    return { name: 'min-efficiency', status: 'inconclusive', detail: 'No trustworthy task-level evidence to measure busy core time.' };
   }
   const model = computeEfficiencyModel({
     app: appModel.app, stages: appModel.stages,
     executorsAdded: appModel.executors.added, runAggregates: appModel.runAggregates,
   });
   if (model.wastagePct == null) {
-    return { name: 'min-efficiency', status: 'inconclusive', detail: 'Efficiency could not be computed (no available compute hours).' };
+    return { name: 'min-efficiency', status: 'inconclusive', detail: 'Busy core time could not be computed (no available compute hours).' };
   }
-  const efficiencyPct = 100 - model.wastagePct;
-  return efficiencyPct < minPct
-    ? { name: 'min-efficiency', status: 'violation', detail: `Efficiency ${efficiencyPct}% below budget ${minPct}%.` }
-    : { name: 'min-efficiency', status: 'pass', detail: `Efficiency ${efficiencyPct}% meets budget ${minPct}%.` };
+  const busyCorePct = 100 - model.wastagePct;
+  return busyCorePct < minPct
+    ? { name: 'min-efficiency', status: 'violation', detail: `Busy core time ${busyCorePct}% below budget ${minPct}%.` }
+    : { name: 'min-efficiency', status: 'pass', detail: `Busy core time ${busyCorePct}% meets budget ${minPct}%.` };
 }
 
 function checkRegression(comparison                   , maxRegressionPct        , regressionMetric        )               {
@@ -158,13 +162,16 @@ function pushComparisonBudget(
   results.push(comparison ? check(comparison) : { name, status: 'inconclusive', detail: 'No baseline comparison available to evaluate this budget.' });
 }
 
-export function evaluateBudgets({ appModel, catalog, budgets, comparison }   
+/** `thresholds`: the overrides the catalog was analyzed with, so a budget that recomputes a
+ * detector's figure (--max-skew) uses the same thresholds. */
+export function evaluateBudgets({ appModel, catalog, budgets, comparison, thresholds }   
                                                                                                  
+                                  
  )                                                                        {
   const results                 = [];
   if (Number.isFinite(budgets.maxRuntimeMs)) results.push(checkRuntime(appModel, budgets.maxRuntimeMs ));
   if (Number.isFinite(budgets.maxSpillGb)) results.push(checkSpill(appModel, catalog, budgets.maxSpillGb ));
-  if (Number.isFinite(budgets.maxSkewRatio)) results.push(checkSkew(appModel, budgets.maxSkewRatio ));
+  if (Number.isFinite(budgets.maxSkewRatio)) results.push(checkSkew(appModel, budgets.maxSkewRatio , skewMinTasksForP95(thresholds)));
   if (Number.isFinite(budgets.maxFailedTaskRatePct)) results.push(checkFailedTaskRate(appModel, catalog, budgets.maxFailedTaskRatePct ));
   if (Number.isFinite(budgets.minEfficiencyPct)) results.push(checkEfficiency(appModel, budgets.minEfficiencyPct ));
   // Guarded here so any evaluateBudgets caller benefits: regressionMetric without
@@ -181,6 +188,12 @@ export function evaluateBudgets({ appModel, catalog, budgets, comparison }
     pushComparisonBudget(results, comparison, 'fail-on-introduced',
       (c) => checkFailOnIntroduced(c, budgets.failOnIntroduced ));
   }
+  // Always checked, unlike the opt-in budgets above: a run with no ApplicationEnd is
+  // inconclusive by default, so a passing budget can't hide a truncated log.
+  const incompleteRunFinding = catalog.find((f) => f.type === 'incompleteRun');
+  if (incompleteRunFinding) {
+    results.push({ name: 'run-complete', status: 'inconclusive', detail: incompleteRunFinding.recommendation ?? 'Event log has no ApplicationEnd event.' });
+  }
   return {
     results,
     violated: results.some((r) => r.status === 'violation'),

package/vendor-core/cli/collect-run.js

@@ -1,4 +1,4 @@
-import { readFileSync, readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
+import { readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
 import { join, basename } from 'node:path';
 import { createState, runParse, runParseFiles, reassembleRollingEntries } from '../parser-worker.js';
 import { nodeParseCodecs } from './native-zstd.js';
@@ -6,10 +6,13 @@ import { createModelCallbacks } from '../model-assembler.js';
 import { routeMessage,                     } from '../ingest.js';
                                             
 
+// No whole-file arrayBuffer(): the parser only ever reads bounded slices, and a
+// whole-file read is what capped local logs at 2 GiB.
                            
                              
                                                                              
-                                      
+                                                                                   
+                
  
 
 export function emptyAppModel()           {
@@ -24,18 +27,48 @@ export function emptyAppModel()           {
   };
 }
 
-// File-like shape runParse/runParseFiles need: name, size, slice().arrayBuffer(), arrayBuffer().
+// File-like shape runParse/runParseFiles need (name, size, slice().arrayBuffer()), read with
+// positioned readSync so only the requested slice is ever in memory, whatever the file size.
+// The descriptor opens on the first read and closes once a read reaches the end of the file,
+// so a rolling directory's parts hold at most one open descriptor at a time while streaming.
+// A later read (a zip archive reads its tail first) reopens it; callers must still call
+// close() once parsing settles, to cover reads that stopped early on an error.
 export function nodeFileFromPath(path        )           {
-  const bytes = readFileSync(path);
-  const u8 = new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const name = basename(path);
+  const size = statSync(path).size;
+  let fd                = null;
+  const close = () => {
+    if (fd === null) return;
+    const open = fd;
+    fd = null;
+    closeSync(open);
+  };
   return {
-    name: basename(path),
-    size: u8.length,
+    name,
+    size,
     slice(start        , end        ) {
-      const view = u8.subarray(start, end);
-      return { async arrayBuffer() { return view.slice().buffer; } };
+      return {
+        async arrayBuffer()                       {
+          const from = Math.max(0, start);
+          const to = Math.min(end, size);
+          if (to <= from) return new ArrayBuffer(0);
+          const buf = new Uint8Array(to - from);
+          fd ??= openSync(path, 'r');
+          // readSync may return fewer bytes than asked; loop until the slice is full.
+          for (let filled = 0; filled < buf.length;) {
+            const n = readSync(fd, buf, filled, buf.length - filled, from + filled);
+            if (n === 0) {
+              close();
+              throw new Error(`"${name}" shrank while it was being read`);
+            }
+            filled += n;
+          }
+          if (to === size) close();
+          return buf.buffer;
+        },
+      };
     },
-    async arrayBuffer() { return u8.slice().buffer; },
+    close,
   };
 }
 
@@ -107,26 +140,38 @@ export function collectViaDispatch(
 
 export async function collectRun(inputPath        )                                                        {
   const stat = statSync(inputPath);
-  return collectViaDispatch((state, emit, reject) => {
-    if (stat.isDirectory()) {
-      if (!isRollingLogDirectory(inputPath)) {
-        reject(new Error("This isn't a Spark rolling event-log directory. Pass a single event-log file instead."));
-        return;
-      }
-      const names = readdirSync(inputPath);
-      let ordered;
-      try {
-        ordered = reassembleRollingEntries(names);
-      } catch (e) {
-        reject(e);
-        return;
+  // Every file opened for this run, closed once parsing settles on any path (done, parse
+  // error, decode error, or a throw past the parser's guards).
+  const opened             = [];
+  const open = (path        ) => {
+    const file = nodeFileFromPath(path);
+    opened.push(file);
+    return file;
+  };
+  try {
+    return await collectViaDispatch((state, emit, reject) => {
+      if (stat.isDirectory()) {
+        if (!isRollingLogDirectory(inputPath)) {
+          reject(new Error("This isn't a Spark rolling event-log directory. Pass a single event-log file instead."));
+          return;
+        }
+        const names = readdirSync(inputPath);
+        let ordered;
+        try {
+          ordered = reassembleRollingEntries(names);
+        } catch (e) {
+          reject(e);
+          return;
+        }
+        const files = ordered.map((name) => open(join(inputPath, name)));
+        // .catch(reject), not void: a throw past the parser's guards would otherwise leave
+        // this Promise pending forever, surfacing only as an unhandled rejection.
+        runParseFiles(files, state, { emit, ...nodeParseCodecs }).catch(reject);
+      } else {
+        runParse(open(inputPath), state, { emit, ...nodeParseCodecs }).catch(reject);
       }
-      const files = ordered.map((name) => nodeFileFromPath(join(inputPath, name)));
-      // .catch(reject), not void: a throw past the parser's guards would otherwise leave
-      // this Promise pending forever, surfacing only as an unhandled rejection.
-      runParseFiles(files, state, { emit, ...nodeParseCodecs }).catch(reject);
-    } else {
-      runParse(nodeFileFromPath(inputPath), state, { emit, ...nodeParseCodecs }).catch(reject);
-    }
-  }, (msg) => new Error((msg                       ).message));
+    }, (msg) => new Error((msg                       ).message));
+  } finally {
+    for (const file of opened) file.close();
+  }
 }