sparkforensics-server 0.5.2 → 0.6.0

package/public/favicon.svg

@@ -1,4 +1,4 @@
 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32">
-  <rect width="32" height="32" rx="7" fill="#181614"/>
-  <path d="M17.5 5 8 18h6.5l-1 9L24 14h-6.5l1-9z" fill="#ff6b2c"/>
+  <rect width="32" height="32" rx="6" fill="#4f46e5"/>
+  <path d="M17.5 5 8 18h6.5l-1 9L24 14h-6.5l1-9z" fill="#eceafd"/>
 </svg>

package/public/index.html

@@ -10,9 +10,6 @@
     <meta property="og:url" content="https://shuffle-works.github.io/sparkforensics/">
     <meta property="og:title" content="SparkForensics">
     <meta property="og:description" content="Zero-backend browser dashboard that parses Apache Spark event logs to diagnose performance bottlenecks: skew, spill, GC pressure, stragglers, cache utilization, and more.">
-    <link rel="preconnect" href="https://fonts.googleapis.com">
-    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-    <link href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
     <script>
       try {
         var stored = localStorage.getItem('shuffle-works-theme');
@@ -23,14 +20,14 @@
           document.documentElement.setAttribute('data-theme', 'light');
       } catch (_) {}
     </script>
-    <script type="module" crossorigin src="./assets/index-B-03mG7K.js"></script>
+    <script type="module" crossorigin src="./assets/index-DwqkFUFk.js"></script>
     <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
     <link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
     <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-BYl-AzN5.js">
-    <link rel="modulepreload" crossorigin href="./assets/react-vendor-DEdfrRTJ.js">
-    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-ZpE6CkHx.js">
+    <link rel="modulepreload" crossorigin href="./assets/react-vendor-DfpuPQZf.js">
+    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-OO7X1HAx.js">
     <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-DLioOiRN.css">
-    <link rel="stylesheet" crossorigin href="./assets/index-2h_TgUAb.css">
+    <link rel="stylesheet" crossorigin href="./assets/index-DhpRGMZc.css">
   </head>
   <body>
     <div id="root"></div>

package/vendor-core/allocation.js

@@ -1,7 +1,9 @@
+import { computePeakConcurrentExecutorCount } from './core-count.js';
 import { parseSparkMemoryMB } from './spark-memory.js';
-                                                                                         
+                                                      
 
 const MS_PER_HOUR = 3_600_000;
+const MS_PER_SECOND = 1000;
 const MIB_PER_GIB = 1024;
 // Spark's documented floor and factor for the default executor memory overhead
 // (spark.executor.memoryOverhead = max(factor * executor memory, 384 MiB)).
@@ -14,12 +16,29 @@ const DEFAULT_OVERHEAD_FACTOR = 0.1;
                                                                                              
                                                         
                                
+                                                                                               
+                                                                                              
+                                         
+                                                                                     
+                               
+                                                                                               
+                                                                                                    
+                               
+                                                                                               
+                               
+                                                                                     
+                                 
  
 
+// The fields the allocation reads from the parsed model's executor events, so a detector's
+// narrower event shapes qualify too.
+                                                                                      
+                                                                   
+
                                   
-                           
-                             
+                                                                     
                                                                   
+                                                                    
  
 
 /** The latest timestamp the log records: the application start and end and every stage and
@@ -43,7 +62,7 @@ const DEFAULT_EXECUTOR_MEMORY_MIB = 1024;
 // plus overhead, plus off-heap and PySpark worker memory when configured. Null when the log
 // records no Spark properties (nothing to tell a default from a missing config) or a memory key it
 // does record cannot be read.
-function executorMemoryMiB(app                     )                {
+function executorMemoryMiB(app                        )                {
   const config = app?.config;
   if (config == null) return null;
   // undefined: key absent; null: present but unreadable.
@@ -63,6 +82,11 @@ function executorMemoryMiB(app                     )                {
   return heap + overhead + offHeap + pyspark;
 }
 
+function loggedDynamicAllocation(app                        )                      {
+  const raw = app?.config?.['spark.dynamicAllocation.enabled'];
+  return raw == null ? null : raw.trim().toLowerCase() === 'true' ? 'on' : 'off';
+}
+
 /** Allocated core-hours and memory GiB-hours from the executor lifecycle: each executor counts
  * from its ExecutorAdded timestamp to its first later ExecutorRemoved timestamp. One with no
  * removal closes at the application end when the log has one, else at the last timestamp the log
@@ -72,35 +96,96 @@ function executorMemoryMiB(app                     )                {
  * else the larger of 384 MiB and spark.executor.memoryOverheadFactor, default 0.1, times the
  * memory), plus spark.memory.offHeap.size when spark.memory.offHeap.enabled is true, plus
  * spark.executor.pyspark.memory.
+ * The executor counts and seconds come from the same alive intervals.
  * Null, never 0, for a figure whose inputs the log lacks. */
 export function computeAllocation(input                 )             {
-  const added = input.executors.added.filter((e)                          => e.kind === 'added');
-  if (added.length === 0) return { coreHours: null, memoryGbHours: null };
-  const removedAt = new Map                  ();
-  for (const e of input.executors.removed) {
-    if (e.kind !== 'removed') continue;
-    const times = removedAt.get(e.executorId);
-    if (times) times.push(e.timestamp); else removedAt.set(e.executorId, [e.timestamp]);
+  const dynamicAllocation = loggedDynamicAllocation(input.app);
+  const alive = aliveIntervals(input);
+  if (alive.length === 0) {
+    return { coreHours: null, memoryGbHours: null, dynamicAllocation, executorsPeak: null, executorsMean: null, executorCores: null, executorSeconds: null };
   }
   const closeAt = input.app?.endTime ?? lastObservedTimestamp(input);
-  const configuredCores = Number.parseInt(input.app?.config?.['spark.executor.cores'] ?? '', 10);
   const memoryMiB = executorMemoryMiB(input.app);
 
   let coreMs = 0;
   let memoryMiBMs = 0;
+  let aliveMsTotal = 0;
   let coresKnown = true;
-  const seen = new Set        ();
-  for (const e of added) {
-    if (seen.has(e.executorId)) continue; // a replayed ExecutorAdded is the same executor
-    seen.add(e.executorId);
-    const removal = (removedAt.get(e.executorId) ?? []).filter((t) => t >= e.timestamp).sort((a, b) => a - b)[0];
-    const aliveMs = Math.max(0, (removal ?? closeAt ?? e.timestamp) - e.timestamp);
-    const cores = e.totalCores > 0 ? e.totalCores : Number.isFinite(configuredCores) ? configuredCores : null;
-    if (cores == null) coresKnown = false; else coreMs += cores * aliveMs;
+  const coreCounts = new Set        ();
+  for (const { start, end, cores } of alive) {
+    const aliveMs = end - start;
+    aliveMsTotal += aliveMs;
+    if (cores == null) coresKnown = false; else { coreMs += cores * aliveMs; coreCounts.add(cores); }
     memoryMiBMs += (memoryMiB ?? 0) * aliveMs;
   }
+  const startTime = input.app?.startTime;
+  const windowMs = closeAt != null && startTime != null ? closeAt - startTime : 0;
+  const firstRemovals = alive.flatMap((a) => (a.removedAt == null ? [] : [{ executorId: a.executorId, timestamp: a.removedAt }]));
   return {
     coreHours: coresKnown ? coreMs / MS_PER_HOUR : null,
     memoryGbHours: memoryMiB != null ? memoryMiBMs / MIB_PER_GIB / MS_PER_HOUR : null,
+    dynamicAllocation,
+    executorsPeak: computePeakConcurrentExecutorCount(alive.map((a) => ({ executorId: a.executorId, timestamp: a.start })), firstRemovals),
+    executorsMean: windowMs > 0 ? aliveMsTotal / windowMs : null,
+    executorCores: coresKnown && coreCounts.size === 1 ? [...coreCounts][0] : null,
+    executorSeconds: aliveMsTotal / MS_PER_SECOND,
   };
 }
+
+                         
+                     
+                
+              
+                           
+                                                                                         
+                       
+ 
+
+// Each executor's alive interval: from its ExecutorAdded timestamp to its first later
+// ExecutorRemoved timestamp, else the application end, else the last timestamp the log records.
+// The one definition computeAllocation and allocatedCoreMsWithin share.
+function aliveIntervals(input                 )                  {
+  const removedAt = new Map                  ();
+  for (const e of input.executors.removed) {
+    const times = removedAt.get(e.executorId);
+    if (times) times.push(e.timestamp); else removedAt.set(e.executorId, [e.timestamp]);
+  }
+  const closeAt = input.app?.endTime ?? lastObservedTimestamp(input);
+  const configuredCores = Number.parseInt(input.app?.config?.['spark.executor.cores'] ?? '', 10);
+  const out                  = [];
+  const seen = new Set        ();
+  for (const e of input.executors.added) {
+    if (seen.has(e.executorId)) continue; // a replayed ExecutorAdded is the same executor
+    seen.add(e.executorId);
+    const removal = (removedAt.get(e.executorId) ?? []).filter((t) => t >= e.timestamp).sort((a, b) => a - b)[0] ?? null;
+    out.push({
+      executorId: e.executorId,
+      start: e.timestamp,
+      end: Math.max(e.timestamp, removal ?? closeAt ?? e.timestamp),
+      removedAt: removal,
+      cores: e.totalCores != null && e.totalCores > 0 ? e.totalCores : Number.isFinite(configuredCores) ? configuredCores : null,
+    });
+  }
+  return out;
+}
+
+/** Allocated core-milliseconds: the figure behind `coreHours`, the capacity idle-core figures are
+ * measured against. Null when the log has no executor events or an executor's cores are unknown. */
+export function allocatedCoreMs(input                 )                {
+  const { coreHours } = computeAllocation(input);
+  return coreHours != null && coreHours > 0 ? coreHours * MS_PER_HOUR : null;
+}
+
+/** The part of the allocated core-milliseconds held inside `windows` (disjoint, sorted
+ * intervals, such as the union of the stages' windows), counted from the same alive intervals as
+ * `allocatedCoreMs`. Null when `allocatedCoreMs` is. */
+export function allocatedCoreMsWithin(input                 , windows                         )                {
+  if (allocatedCoreMs(input) == null) return null;
+  let coreMs = 0;
+  for (const { start, end, cores } of aliveIntervals(input)) {
+    for (const [from, to] of windows) {
+      coreMs += (cores ?? 0) * Math.max(0, Math.min(end, to) - Math.max(start, from));
+    }
+  }
+  return coreMs;
+}

package/vendor-core/api-merge-writes.js

@@ -74,6 +74,7 @@ export function apiMergeWrites(sql                             )
       kind: resolved?.kind ?? null,
       target: resolved?.target ?? null,
       outputRows: null,
+      mergeRows: null,
       raw: `Delta MERGE operation, executions ${span}${group.unclean ? ` (${group.unclean})` : ''}`,
     };
   });

package/vendor-core/core-source-hash.txt

@@ -1 +1 @@
-7aaca3af812e0e5a4f4632ad7577fc438b589520a27422ce8c986b156de82260
+9c33274bcae34f425c7d625bdea9cfacdb400f333bc3a5ced9c86a0e75094916

package/vendor-core/delta-log-paths.js

@@ -9,6 +9,15 @@ import { isCut,                  } from './write-target-args.js';
 // scans and `DeltaLogFileIndex [<path>/_delta_log/<file>, ...]`.
 const DELTA_LOG_PATH = /((?:[A-Za-z][A-Za-z0-9+.-]*:\/{1,3}|\/)[^\s,[\]()#]+?)\/_delta_log(?=[/[\],\s)]|$)/g;
 
+// Hadoop prints a local path as `file:/p` and a URI as `file:///p`: one table, two spellings.
+// Both read as `file:///p` so a table is not counted twice. A `file://host/p` is left alone.
+const LOCAL_FILE_PREFIX = /^file:(?:\/|\/\/\/)(?=[^/])/;
+
+/** The one spelling of a table path: the two local-file forms agree, other paths pass through. */
+export function normalizeTablePath(path        )         {
+  return path.replace(LOCAL_FILE_PREFIX, 'file:///').replace(/\/+$/, '');
+}
+
                                                               
 
 // Every distinct table path named in the plans; `unreliable` when a plan is missing or a path cut.
@@ -26,7 +35,7 @@ function deltaLogPaths(executions                                  )           {
         if (matches.length < mentions) unreliable = true;
         for (const m of matches) {
           if (isCut(m[1])) unreliable = true;
-          else paths.add(m[1].replace(/\/+$/, ''));
+          else paths.add(normalizeTablePath(m[1]));
         }
       }
     }, { dedupe: true });

package/vendor-core/detectors.js

@@ -1,7 +1,8 @@
-import { pathBasename, formatBytes, IMPACT_BAND_ORDER } from './format-utils.js';
+import { pathBasename, formatBytes, IMPACT_BAND_ORDER, MS_PER_CORE_HOUR } from './format-utils.js';
 import { medianOfSorted } from './median.js';
 import { shareLabel } from './finding-presentation.js';
 import { scanRelationId } from './plan-summary.js';
+import { allocatedCoreMs, computeAllocation } from './allocation.js';
 import { computePeakConcurrentCores, computePeakConcurrentExecutorCount } from './core-count.js';
 import { walkPlanTree } from './plan-tree-walk.js';
 import { computeCoreLocalityRatio } from './core-locality-ratio.js';
@@ -19,7 +20,7 @@ import { totalExecutorCpuMs } from './run-totals.js';
 import { DUPLICATE_SUBTREE_DIFFERING_NOTE, duplicateSubtreeDetail, SLOW_HOST_DIMENSION_LABEL } from './finding-generic-recommendation.js';
 import { stageIdsForSqlExec } from './sql-stages.js';
 import { cyrb53 } from './string-hash.js';
-import { decreaseConf, increaseConf, setConf } from './remediation.js';
+import { codeFix, decreaseConf, increaseConf, setConf } from './remediation.js';
 import { MAX_FAILURE_GROUPS, describeTaskFailure,                       } from './task-failure.js';
                                                                                                
                                                                                                                                                                                      
@@ -742,9 +743,9 @@ function skewFix(stage               , ctx             , shuffleEvidence = false
       remediation: [decreaseConf('spark.sql.files.maxPartitionBytes')],
     };
   }
-  if (origin === 'other') return { origin, text: SKEW_KEY_REMEDY, remediation: [] };
+  if (origin === 'other') return { origin, text: SKEW_KEY_REMEDY, remediation: [codeFix(SKEW_KEY_REMEDY)] };
   // Before Spark 3.0 there is no AQE skew-join handling, so the stage gets the generic advice.
-  if (predatesAqeSkewJoin(ctx.app)) return { origin: 'other', text: SKEW_KEY_REMEDY, remediation: [] };
+  if (predatesAqeSkewJoin(ctx.app)) return { origin: 'other', text: SKEW_KEY_REMEDY, remediation: [codeFix(SKEW_KEY_REMEDY)] };
   return { origin, ...skewJoinFix(ctx.app) };
 }
 
@@ -756,9 +757,11 @@ function skewJoinFix(app                    )
       remediation: [setConf('spark.sql.adaptive.enabled', true), ...setConfUnlessLogged(app, key, true)],
     };
   }
-  return switchFix(loggedAs(app, key, true), key, true,
+  const fix = switchFix(loggedAs(app, key, true), key, true,
     `for join-driven skew, enable AQE skew-join handling (${key}); otherwise ${SKEW_KEY_REMEDY}`,
     `AQE skew-join handling is already on, so ${SKEW_KEY_REMEDY}`);
+  // With the switch already on, only the key is left to fix: a change to the job, not a property.
+  return fix.remediation.length === 0 ? { ...fix, remediation: [codeFix(SKEW_KEY_REMEDY)] } : fix;
 }
 
 // The resources flag is read from the same property as the logged conf.
@@ -767,6 +770,23 @@ function dynamicAllocationFix(app             , recommend        , alreadyOn
   return switchFix(app.resources?.dynamicAllocationEnabled === true || loggedAs(app, key, true), key, true, recommend, alreadyOn);
 }
 
+// Idle capacity is cured by a smaller cluster, which the run's dynamic allocation decides how to
+// size. With it on: the cap on how many executors it scales to, and the floor it holds when that
+// is logged above 0 (both lowered together). With it off or unset: the alternatives of a fixed
+// executor count or switching it on, which dynamicAllocationFix already names, so the
+// recommendation reads "either ... or" and a consumer applies one entry. Not executorIdleTimeout:
+// autoscalingChurn recommends raising it, so lowering it here would contradict that finding.
+function idleCapacityFix(app             , recommend        , alreadyOnLead        )                                               {
+  const fix = dynamicAllocationFix(app, recommend, alreadyOnLead);
+  // dynamicAllocationFix's remediation is empty exactly when dynamic allocation is already on.
+  if (fix.remediation.length > 0) return { ...fix, remediation: [...fix.remediation, decreaseConf('spark.executor.instances')] };
+  const holdsFloor = Number.parseInt(app.config?.['spark.dynamicAllocation.minExecutors'] ?? '', 10) > 0;
+  return holdsFloor
+    ? { text: `${alreadyOnLead} by lowering spark.dynamicAllocation.maxExecutors and spark.dynamicAllocation.minExecutors`,
+        remediation: [decreaseConf('spark.dynamicAllocation.maxExecutors'), decreaseConf('spark.dynamicAllocation.minExecutors')] }
+    : { text: `${alreadyOnLead} by lowering spark.dynamicAllocation.maxExecutors`, remediation: [decreaseConf('spark.dynamicAllocation.maxExecutors')] };
+}
+
 // The run's effective spark.sql.shuffle.partitions (logged, else Spark's 200) as a count, or null
 // when the logged value is not a count.
 function effectiveShufflePartitions(app                    )                {
@@ -1385,6 +1405,7 @@ export const DETECTORS = [
       const stage = ctx.stages.get(finding.stageId);
       if (!stage) return null;
       let wasteMs = 0;
+      let longestTaskAfterFixMs = 0;
       if (finding.rule === 'maxPartitionTooBig') {
         wasteMs = ((stage.shuffleReadMax ?? 0) / SHUFFLE_THROUGHPUT_BPS) * 1000;
       } else if (finding.rule === 'shufflePartitionSkew') {
@@ -1400,11 +1421,21 @@ export const DETECTORS = [
           // work, not the scheduling cost of tasks you'd add (adding tasks incurs overhead, recovers
           // nothing). Model the achievable duration at target parallelism by scaling down proportionally.
           wasteMs = stageDurationMs * (1 - taskCount / targetTaskCount);
+          // The fix splits the long tasks, so the longest task is the quantity it shortens: the
+          // occupancy clip must not floor the claim at it (TAIL_CLAIM). An even split leaves the
+          // longest task at taskCount / targetTaskCount of today's: a modeled figure.
+          const longestTaskMs = stage.taskDurationMax ?? 0;
+          longestTaskAfterFixMs = longestTaskMs * taskCount / targetTaskCount;
+          // Splitting only shortens tasks: the stage still takes the time outside its longest task
+          // plus the longest task the split leaves, so the claim is at most the longest task's own
+          // reduction. Without a task-duration max (older snapshots) the stage-wide scaling stands.
+          if (longestTaskMs > 0) wasteMs = Math.min(wasteMs, longestTaskMs - longestTaskAfterFixMs);
         }
       } else {
         return null;
       }
-      return singleStageImpact(wasteMs, finding.stageId, ctx, 'modeled', { value: wasteMs, unit: 'ms' });
+      return singleStageImpact(wasteMs, finding.stageId, ctx, 'modeled', { value: wasteMs, unit: 'ms' },
+        finding.rule === 'lowShuffleParallelism' ? { ...TAIL_CLAIM, longestTaskAfterFixMs } : undefined);
     },
   }),
   defineStageDetector({
@@ -2009,12 +2040,15 @@ export const DETECTORS = [
       if (!app || executorsAdded.length === 0 || app.startTime == null || app.endTime == null) return null;
       const appDuration = app.endTime - app.startTime;
       if (appDuration <= 0) return null;
-      // computePeakConcurrentCores (not executorsAdded.length/computeTotalCores): real concurrent
-      // capacity, not a cumulative sum that double-counts a churned-through executor against its
-      // replacement's (spot preemption, dynamicAllocation replacement).
+      // Capacity is what the run was allocated (cores x time alive, the figure behind
+      // metrics.allocation.coreHours), not peak concurrent cores x the whole run: under dynamic
+      // allocation or late-joining executors that is more than was ever held, and the idle figure
+      // would exceed the allocation. Null without executor cores.
+      const capacityCoreMs = allocatedCoreMs({ app, stages: ctx.stages, executors: { added: executorsAdded, removed: executorsRemoved } });
+      if (capacityCoreMs == null) return null;
+      // Peak concurrent cores (not computeTotalCores, a cumulative sum that double-counts a
+      // churned-through executor against its replacement): the cluster size the finding reports.
       const totalCores = computePeakConcurrentCores(app, executorsAdded, executorsRemoved);
-      if (totalCores <= 0) return null;
-      const capacityCoreMs = totalCores * appDuration;
       // Busy core-time (from the whole-run core-time-series, same signal memoryUtilization's
       // idleCores variant already uses), not executor lifetime: an executor that exists for the
       // whole run but sits fully idle must not score as 100% used. Missing runAggregates (older
@@ -2025,15 +2059,12 @@ export const DETECTORS = [
       if (utilization >= thresholds.minUtil) return null;
 
       // CPU-time-based utilization (sparkMeasure): metric only, no threshold.
-      let cpuUtilizationPct                = null;
-      if (totalCores > 0) {
-        // Null when no stage recorded CPU time (older Spark), the same rule as the CLI metrics block.
-        const cpuMs = totalExecutorCpuMs(ctx.stages.values());
-        cpuUtilizationPct = cpuMs == null ? null : Math.round((cpuMs / (appDuration * totalCores)) * 100);
-      }
+      // Null when no stage recorded CPU time (older Spark), the same rule as the CLI metrics block.
+      const cpuMs = totalExecutorCpuMs(ctx.stages.values());
+      const cpuUtilizationPct = cpuMs == null ? null : Math.round((cpuMs / capacityCoreMs) * 100);
 
       const value = Math.round(utilization * 100);
-      const fix = dynamicAllocationFix(app, 'consider reducing cluster size or enabling dynamic allocation',
+      const fix = idleCapacityFix(app, 'consider either reducing cluster size (spark.executor.instances) or enabling dynamic allocation',
         'dynamic allocation is already on, so consider reducing cluster size');
       return {
         type: 'utilization', stageId: null, impactBand: 'info',
@@ -2041,6 +2072,7 @@ export const DETECTORS = [
         utilizationFraction: utilization,
         appDurationMs: appDuration,
         totalCores,
+        allocatedCoreMs: capacityCoreMs,
         cpuUtilizationPct,
         recommendation: `Average executor utilization was only ${value}%: ${fix.text}.`,
         remediation: fix.remediation,
@@ -2048,13 +2080,17 @@ export const DETECTORS = [
     },
     estimate(finding)                        {
       const fraction = finding.utilizationFraction                      ;
-      const appDurationMs = finding.appDurationMs                      ;
-      const totalCores = finding.totalCores                      ;
-      if (fraction == null || appDurationMs == null || totalCores == null) {
+      const allocatedMs = finding.allocatedCoreMs                      ;
+      if (fraction == null || allocatedMs == null) {
         return costOnly('measured');
       }
-      const idleCoreHours = (1 - fraction) * appDurationMs * totalCores / 3.6e6;
-      return costOnly('measured', { value: idleCoreHours, unit: 'coreHours', idle: true });
+      // The same allocated-minus-busy figure as `rawWaste`, in core-milliseconds, next to (never in)
+      // coreTimeMs: that field is busy task time a fix removes, this is capacity no task used.
+      const idleMs = (1 - fraction) * allocatedMs;
+      return {
+        ...costOnly('measured', { value: idleMs / MS_PER_CORE_HOUR, unit: 'coreHours', idle: true }),
+        idleCoreTimeMs: { low: idleMs, high: idleMs },
+      };
     },
   }),
   defineAppDetector({
@@ -2079,25 +2115,26 @@ export const DETECTORS = [
       // cumulative sum or count double-counts a churned-through executor against its replacement's
       // (spot preemption, dynamicAllocation replacement), inflating idle-rate and waste-model figures.
       const peakExecutors = computePeakConcurrentExecutorCount(executorsAdded, executorsRemoved);
-      const totalCores = computePeakConcurrentCores(app, executorsAdded, executorsRemoved);
-      // Hoisted above 1a (also 1b/1c's input) so the idle-cores finding carries the allocated
-      // memory its MB-seconds estimate needs.
       const allocatedMB = app.resources?.executor?.memoryMB ?? null;
 
       // ── 1a idle-cores rate ────────────────────────────────────────────────
       const busyCoreMs = runAggregates?.busyCoreMs;
-      if (busyCoreMs != null && totalCores > 0) {
-        const capacityCoreMs = totalCores * appDurationMs;
-        const idleRate = capacityCoreMs > 0 ? 1 - (busyCoreMs / capacityCoreMs) : 0;
+      // Allocated core-time, as for `utilization`, so the two findings of one idle condition agree.
+      const allocationInput = { app, stages, executors: { added: executorsAdded, removed: executorsRemoved } };
+      const capacityCoreMs = allocatedCoreMs(allocationInput);
+      if (busyCoreMs != null && capacityCoreMs != null) {
+        const idleRate = 1 - (busyCoreMs / capacityCoreMs);
         if (idleRate > thresholds.idleCoreWarn) {
           const value = Math.round(idleRate * 100);
-          const fix = dynamicAllocationFix(app, 'reduce cluster size or enable dynamic allocation',
+          const { memoryGbHours } = computeAllocation(allocationInput);
+          const fix = idleCapacityFix(app, 'either reduce cluster size (spark.executor.instances) or enable dynamic allocation',
             'dynamic allocation is already on, so reduce cluster size');
           out.push({
             type: 'memoryUtilization', variant: 'idleCores', stageId: null,
             impactBand: 'warning', metric: 'idleCoreRate', value,
-            // Raw (unrounded) rate plus sizing inputs for the impact estimator: `value` is rounded pct.
-            idleRateFraction: idleRate, allocatedMB, peakExecutors, appDurationMs,
+            // Raw (unrounded) rate plus the allocated memory-time for the impact estimator: `value` is rounded pct.
+            idleRateFraction: idleRate,
+            allocatedMBSeconds: memoryGbHours != null ? memoryGbHours * 1024 * 3600 : null,
             recommendation: `${value}% of available core-time ran no task: ${fix.text}.`,
             remediation: fix.remediation,
           });
@@ -2183,13 +2220,11 @@ export const DETECTORS = [
         return costOnly('measured', { value: finding.value, unit: 'mbSeconds' });
       }
       if (finding.variant === 'idleCores') {
-        // Idle core-time priced as memory held but unused: the same MB-seconds unit as wasteModel, so comparable.
+        // Idle core-time priced as memory held but unused: the allocated memory-seconds times the idle rate.
         const idleRateFraction = finding.idleRateFraction                      ;
-        const allocatedMB = finding.allocatedMB                      ;
-        const peakExecutors = finding.peakExecutors                      ;
-        const appDurationMs = finding.appDurationMs                      ;
-        if (idleRateFraction != null && allocatedMB != null && peakExecutors != null && appDurationMs != null) {
-          const wastedMBSeconds = idleRateFraction * allocatedMB * peakExecutors * (appDurationMs / 1000);
+        const allocatedMBSeconds = finding.allocatedMBSeconds                             ;
+        if (idleRateFraction != null && allocatedMBSeconds != null) {
+          const wastedMBSeconds = idleRateFraction * allocatedMBSeconds;
           return costOnly('modeled', { value: wastedMBSeconds, unit: 'mbSeconds' });
         }
         return costOnly('modeled');

package/vendor-core/docs-content/detection/mem.md

@@ -1,7 +1,7 @@
 ### `MEM`: Memory utilization {#mem}
 
 Executor memory or core capacity may be over- or under-provisioned: more
-than 50% of available core time ran no task, an executor's heap peaked above
+than 50% of allocated core time ran no task, an executor's heap peaked above
 95% of its allocation, or it stayed below 70%. Some
 detail here needs `spark.eventLog.logStageExecutorMetrics=true` on the run
 being analyzed; without it, per-executor memory usage can't be broken down.

package/vendor-core/docs-content/detection/skew.md

@@ -8,7 +8,8 @@ stage. The fix depends on what the stage reads, which the finding's
 already has it; otherwise salt the key or repartition on a better key. A
 stage that reads files with uneven sizes (`inputScan`) gets compaction of
 small files or a lower `spark.sql.files.maxPartitionBytes`. Any other stage
-(`other`) gets the salting advice and no conf. Flagged when P95 task time
+(`other`) gets the salting advice and no conf, as a `code` entry in
+`remediation`; so does a join stage whose skew-join handling is already on. Flagged when P95 task time
 (the longest task, on a stage with fewer than 20 tasks) exceeds 3x the median
 and the recoverable tail is at least 0.5% of the run. The median is the
 textbook one: on an even task count, the mean of the two middle values.

package/vendor-core/docs-content/detection/util.md

@@ -1,4 +1,8 @@
 ### `UTIL`: Low utilization {#util}
 
-Allocated executors sit idle for a large share of the application run.
-Consider a smaller cluster, or enable dynamic allocation.
+Allocated executors sit idle for a large share of the application run: under
+60% of the core time the run allocated (cores times the time each executor was
+alive) ran a task. Consider a smaller cluster. With dynamic allocation off,
+either lower `spark.executor.instances` or enable dynamic allocation; the two are
+alternatives. With it on, lower `spark.dynamicAllocation.maxExecutors`, and
+`spark.dynamicAllocation.minExecutors` too when the run sets it above 0.

package/vendor-core/efficiency-model.js

@@ -1,12 +1,13 @@
 // §5 Efficiency/wastage model. DESIGN SPIKE:
 // decomposes available compute-hours into driver-bound vs executor-bound waste, plus two floors.
-import { computeWallClock } from './wall-clock.js';
+import { stageWindows } from './wall-clock.js';
+import { allocatedCoreMs, allocatedCoreMsWithin } from './allocation.js';
 import { computePeakConcurrentCores } from './core-count.js';
                                                                              
 
 export function computeEfficiencyModel({ app, stages, executorsAdded, executorsRemoved, runAggregates }   
                            
-                               
+                                                                      
                                   
                                     
                                       
@@ -18,23 +19,25 @@ export function computeEfficiencyModel({ app, stages, executorsAdded, executorsR
                           
                                               
   {
-  // Capacity is the peak concurrent core count, the one the utilization and idle-cores detectors
-  // use, so the Unused core time tile matches the verdict's idle figure. Summing every addition
-  // (computeTotalCores) counts a replaced executor's cores alongside its replacement's.
-  // `app ?? {}`: callers tolerate a null app (malformed logs); `app!` would crash on app.resources.
-  const totalCores = computePeakConcurrentCores(app ?? {}, executorsAdded, executorsRemoved);
-  const appDurationMs = (app?.endTime ?? 0) - (app?.startTime ?? 0);
-  const wc = computeWallClock(app, stages                                                               );
+  // Capacity is the run's allocation (cores x time alive, metrics.allocation.coreHours), the one the
+  // utilization and idle-cores detectors use, so the Unused core time tile matches the verdict's
+  // idle figure. Peak concurrent cores times the whole run would count cores the run never held
+  // under dynamic allocation. Null without executor cores: no capacity, so no figure.
+  const input = { app, stages, executors: { added: executorsAdded, removed: executorsRemoved } };
+  const allocatedMs = allocatedCoreMs(input);
+  // Split by where the allocated cores were held, from the same alive intervals: inside the union
+  // of the stages' windows, or outside it (startup, gaps between stages, after the last stage).
+  // Executors that join after startup hold no cores there, so spreading the allocation evenly
+  // over the run would bill the driver for cores that never existed.
+  const allocatedActiveCoreMs = allocatedMs == null ? 0 : allocatedCoreMsWithin(input, stageWindows(stages)) ?? 0;
 
-  const availableComputeHours = totalCores * (appDurationMs / 3600000);
+  const availableComputeHours = (allocatedMs ?? 0) / 3600000;
 
-  // Driver-bound: cores idle during startup + gaps + idle (no active stage).
-  const driverIdleMs = wc.startup + wc.gaps + wc.idle;
-  const driverWasteHours = totalCores * (driverIdleMs / 3600000);
+  // Driver-bound: cores held while no stage was active (startup, gaps, idle).
+  const driverWasteHours = Math.max(0, (allocatedMs ?? 0) - allocatedActiveCoreMs) / 3600000;
 
   // Executor-bound: allocated capacity beyond busy cores during stagesActive. Tasks only run in
   // active windows, so whole-run busyCoreMs already lives inside stagesActive.
-  const allocatedActiveCoreMs = totalCores * wc.stagesActive;
   const busyCoreMs = runAggregates?.busyCoreMs ?? 0;
   const executorWasteHours = Math.max(0, allocatedActiveCoreMs - busyCoreMs) / 3600000;
 
@@ -44,6 +47,11 @@ export function computeEfficiencyModel({ app, stages, executorsAdded, executorsR
   let totalTaskMs = 0;
   const perStage = runAggregates?.perStage ?? {};
   for (const k of Object.keys(perStage)) totalTaskMs += perStage[k].totalTaskDurationSum;
+  // The floor is the least time the work takes on the widest the cluster ever was: peak concurrent
+  // cores (not the allocation), summing every addition (computeTotalCores) would count a replaced
+  // executor's cores alongside its replacement's. `app ?? {}`: callers tolerate a null app
+  // (malformed logs); `app!` would crash on app.resources.
+  const totalCores = computePeakConcurrentCores(app ?? {}, executorsAdded, executorsRemoved);
   const floorZeroSkewMs = totalCores > 0 ? totalTaskMs / totalCores : 0;
 
   let dominantWaste                               = null;

package/vendor-core/finding-presentation.js

@@ -35,7 +35,14 @@ export const shareLabel = (share        )         => `${Math.round(share * 1e6)
 // a generic line reads the same decision and never recommends a switch the row says is on.
 // A finding with no remediation (older or hand-built data) keeps the property wording.
 function switchAlreadyOn(finding                                 , key        )          {
-  return finding.remediation != null && !finding.remediation.some((r) => r.key === key);
+  return finding.remediation != null && !finding.remediation.some((r) => r.kind === 'conf' && r.key === key);
+}
+
+// The properties an idle-capacity finding lowers once dynamic allocation is on, worded as its row
+// does: the executor cap, and the floor when the row's remediation lowers it too.
+function idleCapacityLowering(f                                 )         {
+  const lowersFloor = f.remediation?.some((r) => r.kind === 'conf' && r.key === 'spark.dynamicAllocation.minExecutors');
+  return ` by lowering spark.dynamicAllocation.maxExecutors${lowersFloor ? ' and spark.dynamicAllocation.minExecutors' : ''}`;
 }
 
 const SKEW_JOIN_KEY = 'spark.sql.adaptive.skewJoin.enabled';
@@ -46,7 +53,7 @@ const SKEW_JOIN_AQE_OFF = 'AQE is off, so enable it (spark.sql.adaptive.enabled)
 function skewJoinGeneric(f                                                      , unset        )         {
   if (f.origin === 'inputScan') return 'Uneven input files: compact small files or split large ones (lower spark.sql.files.maxPartitionBytes).';
   if (f.origin === 'other') return 'Work is uneven across tasks: salt the key or repartition on a better key.';
-  if (f.remediation?.some((r) => r.key === 'spark.sql.adaptive.enabled')) return SKEW_JOIN_AQE_OFF;
+  if (f.remediation?.some((r) => r.kind === 'conf' && r.key === 'spark.sql.adaptive.enabled')) return SKEW_JOIN_AQE_OFF;
   return switchAlreadyOn(f, SKEW_JOIN_KEY) ? SKEW_JOIN_ALREADY_ON : unset;
 }
 const DYNAMIC_ALLOCATION_KEY = 'spark.dynamicAllocation.enabled';
@@ -280,8 +287,8 @@ export const FINDING_PRESENTATION
     genericRecommendation(f) {
       switch (f.variant) {
         case 'idleCores': return switchAlreadyOn(f, DYNAMIC_ALLOCATION_KEY)
-          ? 'Dynamic allocation is already on, so reduce cluster size.'
-          : 'Reduce cluster size or enable dynamic allocation.';
+          ? `Dynamic allocation is already on, so reduce cluster size${idleCapacityLowering(f)}.`
+          : 'Either reduce cluster size (spark.executor.instances) or enable dynamic allocation.';
         case 'wasteModel': return 'Review spark.executor.memory and executor count.';
         case 'memoryBand':
           if (f.dataUnavailable) return undefined;
@@ -298,8 +305,8 @@ export const FINDING_PRESENTATION
     thresholdSummary: (t) => `average executor utilization below ${shareLabel(t.minUtil)}`,
     actionLabel: () => 'Reduce cluster size',
     genericRecommendation: (f) => (switchAlreadyOn(f, DYNAMIC_ALLOCATION_KEY)
-      ? 'Dynamic allocation is already on, so consider reducing cluster size.'
-      : 'Consider reducing cluster size or enabling dynamic allocation.'),
+      ? `Dynamic allocation is already on, so consider reducing cluster size${idleCapacityLowering(f)}.`
+      : 'Consider either reducing cluster size (spark.executor.instances) or enabling dynamic allocation.'),
   },
   coreLocality: {
     name: 'core locality',