sparkforensics-server 0.6.0 → 0.8.1

package/public/index.html

@@ -20,14 +20,15 @@
           document.documentElement.setAttribute('data-theme', 'light');
       } catch (_) {}
     </script>
-    <script type="module" crossorigin src="./assets/index-DwqkFUFk.js"></script>
+    <script type="module" crossorigin src="./assets/index-fDiwfuoO.js"></script>
     <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
-    <link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
-    <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-BYl-AzN5.js">
-    <link rel="modulepreload" crossorigin href="./assets/react-vendor-DfpuPQZf.js">
-    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-OO7X1HAx.js">
-    <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-DLioOiRN.css">
-    <link rel="stylesheet" crossorigin href="./assets/index-DhpRGMZc.css">
+    <link rel="modulepreload" crossorigin href="./assets/react-vendor-C6KcUD36.js">
+    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-DK1_iz6J.js">
+    <link rel="modulepreload" crossorigin href="./assets/inertValue-L9VxV0VE.js">
+    <link rel="modulepreload" crossorigin href="./assets/button-DbpTz2yk.js">
+    <link rel="modulepreload" crossorigin href="./assets/composite-C9Nu2EI9.js">
+    <link rel="modulepreload" crossorigin href="./assets/useScrollLock-BCxn_dMp.js">
+    <link rel="stylesheet" crossorigin href="./assets/index-CaRVPvAT.css">
   </head>
   <body>
     <div id="root"></div>

package/vendor-core/allocation.js

@@ -1,14 +1,15 @@
 import { computePeakConcurrentExecutorCount } from './core-count.js';
 import { parseSparkMemoryMB } from './spark-memory.js';
+import { effectiveSparkConf, executorOverheadSettings,                 } from './spark-conf.js';
                                                       
 
 const MS_PER_HOUR = 3_600_000;
 const MS_PER_SECOND = 1000;
 const MIB_PER_GIB = 1024;
 // Spark's documented floor and factor for the default executor memory overhead
-// (spark.executor.memoryOverhead = max(factor * executor memory, 384 MiB)).
-const MIN_OVERHEAD_MIB = 384;
-const DEFAULT_OVERHEAD_FACTOR = 0.1;
+// (spark.executor.memoryOverhead = max(factor * executor memory, 384 MiB)), for a run whose
+// Spark version does not read the factor or minimum properties.
+const OVERHEAD_FALLBACK = { floorMB: 384, floorPct: 0.1 };
 
                              
                                                                                                    
@@ -36,7 +37,7 @@ const DEFAULT_OVERHEAD_FACTOR = 0.1;
                                                                    
 
                                   
-                                                                     
+                                                                                      
                                                                   
                                                                     
  
@@ -55,47 +56,48 @@ export function lastObservedTimestamp(input                 )                {
   return last;
 }
 
-// Spark's default executor memory when spark.executor.memory is unset.
-const DEFAULT_EXECUTOR_MEMORY_MIB = 1024;
-
 // One container's memory in MiB, as Spark requests it from the cluster manager: executor heap,
-// plus overhead, plus off-heap and PySpark worker memory when configured. Null when the log
-// records no Spark properties (nothing to tell a default from a missing config) or a memory key it
-// does record cannot be read.
+// plus overhead, plus off-heap and PySpark worker memory when configured. Every setting is read
+// from the run's effective conf (the logged value, else Spark's default for its version). Null when
+// the log records no Spark properties (nothing to tell a default from a missing config) or a memory
+// key it does record cannot be read.
 function executorMemoryMiB(app                        )                {
   const config = app?.config;
   if (config == null) return null;
+  const layers             = { sparkVersion: app?.sparkVersion, properties: config };
+  const conf = (key        )                     => effectiveSparkConf(layers, key)?.value;
   // undefined: key absent; null: present but unreadable.
-  const mib = (key        )                            => (config[key] == null ? undefined : parseSparkMemoryMB(config[key]));
-  const heap = mib('spark.executor.memory') ?? (config['spark.executor.memory'] == null ? DEFAULT_EXECUTOR_MEMORY_MIB : null);
+  const mib = (key        )                            => {
+    const value = conf(key);
+    return value === undefined ? undefined : parseSparkMemoryMB(value);
+  };
+  const heap = mib('spark.executor.memory');
   if (heap == null) return null;
 
   let overhead = mib('spark.executor.memoryOverhead');
   if (overhead === undefined) overhead = mib('spark.yarn.executor.memoryOverhead'); // legacy key
   if (overhead === undefined) {
-    const factor = Number.parseFloat(config['spark.executor.memoryOverheadFactor'] ?? '');
-    overhead = Math.max(MIN_OVERHEAD_MIB, Math.round(heap * (Number.isFinite(factor) && factor > 0 ? factor : DEFAULT_OVERHEAD_FACTOR)));
+    const { minMB, factor } = executorOverheadSettings(layers, OVERHEAD_FALLBACK);
+    overhead = Math.max(minMB, Math.round(heap * factor));
   }
-  const offHeap = String(config['spark.memory.offHeap.enabled']).toLowerCase() === 'true' ? mib('spark.memory.offHeap.size') ?? 0 : 0;
+  const offHeap = conf('spark.memory.offHeap.enabled')?.toLowerCase() === 'true' ? mib('spark.memory.offHeap.size') ?? 0 : 0;
   const pyspark = mib('spark.executor.pyspark.memory') ?? 0;
   if (overhead === null || offHeap === null || pyspark === null) return null;
   return heap + overhead + offHeap + pyspark;
 }
 
+// The logged `spark.dynamicAllocation.enabled`: a value the app set. Spark's default (off) is not a
+// logged key, so a run that never set it reads null.
 function loggedDynamicAllocation(app                        )                      {
-  const raw = app?.config?.['spark.dynamicAllocation.enabled'];
-  return raw == null ? null : raw.trim().toLowerCase() === 'true' ? 'on' : 'off';
+  const effective = effectiveSparkConf({ sparkVersion: app?.sparkVersion, properties: app?.config }, 'spark.dynamicAllocation.enabled');
+  return effective == null || effective.source === 'default' ? null : effective.value.toLowerCase() === 'true' ? 'on' : 'off';
 }
 
 /** Allocated core-hours and memory GiB-hours from the executor lifecycle: each executor counts
  * from its ExecutorAdded timestamp to its first later ExecutorRemoved timestamp. One with no
  * removal closes at the application end when the log has one, else at the last timestamp the log
  * records (a cut-off log). Cores are the ExecutorAdded event's Total Cores, else
- * spark.executor.cores; memory per executor is spark.executor.memory (default 1g) plus the
- * overhead (spark.executor.memoryOverhead, else the legacy spark.yarn.executor.memoryOverhead,
- * else the larger of 384 MiB and spark.executor.memoryOverheadFactor, default 0.1, times the
- * memory), plus spark.memory.offHeap.size when spark.memory.offHeap.enabled is true, plus
- * spark.executor.pyspark.memory.
+ * spark.executor.cores; memory per executor is the container Spark requests (executorMemoryMiB).
  * The executor counts and seconds come from the same alive intervals.
  * Null, never 0, for a figure whose inputs the log lacks. */
 export function computeAllocation(input                 )             {
@@ -151,7 +153,7 @@ function aliveIntervals(input                 )                  {
     if (times) times.push(e.timestamp); else removedAt.set(e.executorId, [e.timestamp]);
   }
   const closeAt = input.app?.endTime ?? lastObservedTimestamp(input);
-  const configuredCores = Number.parseInt(input.app?.config?.['spark.executor.cores'] ?? '', 10);
+  const configuredCores = Number.parseInt(effectiveSparkConf({ sparkVersion: input.app?.sparkVersion, properties: input.app?.config }, 'spark.executor.cores')?.value ?? '', 10);
   const out                  = [];
   const seen = new Set        ();
   for (const e of input.executors.added) {

package/vendor-core/analyzer.js

@@ -1,11 +1,12 @@
-import { DETECTORS, ENTRY_BY_TYPE,                                                                                     } from './detectors.js';
+import { DETECTORS, ENTRY_BY_TYPE, hasStageOutsideSql, skewThresholdsFor,                                                                                     } from './detectors.js';
 import { effectiveThresholds, findingTunedThresholds, overridesFor, tunedThresholdsNote } from './threshold-overrides.js';
 import { computePeakConcurrentCores } from './core-count.js';
 import { assertNever } from './assert-never.js';
 import { estimateImpact,                  } from './impact-estimator.js';
 import { computeOccupancy,                     } from './occupancy.js';
 import { deriveImpactBand, IMPACT_FLOOR_PCT_CRIT, IMPACT_FLOOR_PCT_WARN,                       } from './impact-band.js';
-import { IMPACT_BAND_ORDER } from './format-utils.js';
+import { IMPACT_BAND_ORDER, MS_PER_CORE_HOUR } from './format-utils.js';
+import { computeWastedCoreHours } from './wasted-core-hours.js';
              
                                                                                                                          
                     
@@ -17,7 +18,7 @@ const detectors                      = DETECTORS;
                                  
                                                                                               
                                                                                                   
-                                                                                                
+                                                                                                                          
                                                                                               
                                                                     
                                   
@@ -33,8 +34,8 @@ function fnv1a(str        )         {
   return (h >>> 0).toString(16).padStart(8, '0');
 }
 
-// The id hash's discriminator slots, in the order it has always joined them: reordering or renaming
-// one changes every finding id, which needs an EVIDENCE_SCHEMA_VERSION bump.
+// The id hash's discriminator slots, in join order: reordering or renaming one changes every
+// finding id, which needs an EVIDENCE_SCHEMA_VERSION bump.
 const DISCRIMINATOR_SLOTS = [
   'host', 'executorId', 'rule', 'variant', 'dimension',
   'direction', 'nodeName', 'rootName', 'subtreeSize', 'groupIndex', 'largerSideBytes',
@@ -43,16 +44,11 @@ const DISCRIMINATOR_SLOTS = [
                                                               
 
 // Per type, the fields that tell apart sibling findings sharing one location and metric; without
-// them those siblings hash to one id:
-//   slowHost (host), memoryUtilization (executorId), partitionSizing (rule);
-//   cacheUtilization (rddId+variant), cachingOpportunity (relation/format for leaf,
-//     operator+relation for composite, executionIds as last resort);
-//   smallFiles (direction/nodeName), duplicatePlanSubtree (groupIndex is the real
-//     uniqueness guarantee: rootName+subtreeSize can collide across groups),
-//     underBroadcast (value+largerSideBytes per node/side).
-// memoryUtilization leaves out `rule`: one heap band per executor, so executorId is already
-// unique and folding rule in risks id churn if band logic changes. partitionSizing keeps
-// `rule`: a stage can emit several rules at once sharing stageId+metric.
+// them those siblings hash to one id (see detector-contract.md, Finding ids). Notes:
+// duplicatePlanSubtree's groupIndex is the real uniqueness guarantee (rootName+subtreeSize can
+// collide across groups); memoryUtilization leaves out `rule` because its one heap band sits on
+// the busiest executor, so executorId is already unique; partitionSizing keeps `rule` because a
+// stage can emit several rules at once sharing stageId+metric.
 const ID_DISCRIMINATORS                                                                              = {
   skew: [], stageShape: ['rule'], shuffle: [], partitionSizing: ['rule'], spill: [], gc: ['direction'],
   slowHost: ['host', 'executorId', 'variant', 'dimension'], stageSlowness: [], stageFailed: ['variant'],
@@ -62,6 +58,8 @@ const ID_DISCRIMINATORS
   cachingOpportunity: ['variant', 'relation', 'format', 'operator', 'executionIds'],
   jobFailureRate: [], configAudit: [],
   duplicatePlanSubtree: ['rootName', 'subtreeSize', 'groupIndex'], smallFiles: ['direction', 'nodeName'],
+  pythonUdf: [],
+  nestedLoopJoin: ['nodeName'],
   underBroadcast: ['largerSideBytes'], overBroadcast: [],
 };
 
@@ -88,35 +86,6 @@ export function findingId(f         )         {
   return fnv1a(`${f.type}|${locationKey(f)}|${f.metric ?? ''}|${f.value ?? f.valueText ?? ''}|${disc}`);
 }
 
-// skew (either branch) and straggler both claim the stage's replayed tail recovery
-// (tailReplayRecoveryMs via tailRecoveryMs): the same slow-task tail reported by two detectors
-// (see "Overlap caveat: skew / straggler" in impact-estimation/caveats-tuning-and-coverage.md). skew's branch only changes
-// the fallback single-task delta on a stage without the replay, so every skew + straggler pair
-// on a stage is flagged. Flags both sides via validationRequired (rather than suppressing
-// either) so neither finding's own diagnostic value is lost; the flag rides the same
-// confidence-caveat UI a reader already sees before trusting either finding's magnitude.
-function overlapNote(otherType                      )         {
-  return `This overlaps with the ${otherType} finding on this stage: both measure the same slow-task tail, so don't add their recoverable-time figures together.`;
-}
-
-function flagSkewStragglerOverlap(findings           )       {
-  const skewStages = new Set(
-    findings.filter((f) => f.type === 'skew' && f.stageId != null).map((f) => f.stageId),
-  );
-  if (skewStages.size === 0) return;
-  const stragglerStages = new Set(
-    findings.filter((f) => f.type === 'straggler' && f.stageId != null).map((f) => f.stageId),
-  );
-  const overlapStages = new Set([...skewStages].filter((id) => stragglerStages.has(id)));
-  if (overlapStages.size === 0) return;
-  for (const f of findings) {
-    if (f.stageId == null || !overlapStages.has(f.stageId)) continue;
-    const note = f.type === 'skew' ? overlapNote('straggler') : f.type === 'straggler' ? overlapNote('skew') : null;
-    if (!note) continue;
-    f.validationRequired = [f.validationRequired, note].filter(Boolean).join(' ');
-  }
-}
-
 function push(out           , entry          , result                            , tuned                         = null)       {
   if (!result) return;
   const detectorVersion = entry.version ?? 1;
@@ -176,6 +145,57 @@ function applySuppression(out           )            {
   return out.filter((f) => f.stageId == null || !dropped.get(f.type)?.has(f.stageId));
 }
 
+// Advice to lower spark.executor.memory (over-provisioned heap, low GC, idle memory-time) contradicts
+// advice to raise it (high GC, spill, a GC-bound straggler tail) where a stage is short of memory.
+// A stage-level lowering finding on the same stage as raising advice is dropped. A run-level one
+// (heap, idle memory-time) stays, since other stages may well have memory to spare, but its advice
+// to lower memory is replaced by a note naming the stages that ask for more.
+const EXECUTOR_MEMORY_KEY = 'spark.executor.memory';
+
+function changesExecutorMemory(f         , direction                         )          {
+  return (f.remediation ?? []).some((r) => r.kind === 'conf' && r.key === EXECUTOR_MEMORY_KEY && r.direction === direction);
+}
+
+function reconcileExecutorMemoryAdvice(out           )            {
+  const raising = out.filter((f) => changesExecutorMemory(f, 'increase'));
+  if (raising.length === 0) return out;
+  const raisingStages = new Set(raising.flatMap((f) => (f.stageId != null ? [f.stageId] : [])));
+  const stageList = [...raisingStages].sort((a, b) => a - b).join(', ');
+  const conflict = raisingStages.size > 0
+    ? `Stage${raisingStages.size === 1 ? '' : 's'} ${stageList} ask${raisingStages.size === 1 ? 's' : ''} for more executor memory, so confirm before lowering it.`
+    : 'Other findings ask for more executor memory, so confirm before lowering it.';
+  const reconciled            = [];
+  for (const f of out) {
+    if (!changesExecutorMemory(f, 'decrease') || changesExecutorMemory(f, 'increase')) { reconciled.push(f); continue; }
+    if (f.stageId != null) {
+      if (!raisingStages.has(f.stageId)) reconciled.push(f);
+      continue;
+    }
+    reconciled.push({
+      ...f,
+      recommendation: `${f.recommendation ?? ''} ${conflict}`.trim(),
+      remediation: (f.remediation ?? []).filter((r) => !(r.kind === 'conf' && r.key === EXECUTOR_MEMORY_KEY && r.direction === 'decrease')),
+    });
+  }
+  return reconciled;
+}
+
+// The run's idle core-ms (the Wasted core-hours figure), the ceiling for one stage's idle claim.
+function runIdleCoreMs(
+  app                     ,
+  executorsAdded                 ,
+  executorsRemoved                 ,
+  runAggregates                      ,
+)                {
+  const { wastedCoreHours } = computeWastedCoreHours(
+    app,
+    executorsAdded                                                                         ,
+    runAggregates,
+    executorsRemoved                                                    ,
+  );
+  return wastedCoreHours == null ? null : wastedCoreHours * MS_PER_CORE_HOUR;
+}
+
 // `app` widened to `SparkAppInfo | null` to match real callers (AppModel.app is
 // nullable at the type level); every detector below tolerates a null app.
 export function analyze(
@@ -203,6 +223,7 @@ export function analyze(
   const impact              = {
     stages, totalCores, sql,
     occupancy: computeOccupancy(stages                                          , totalCores),
+    runIdleCoreMs: runIdleCoreMs(app, executorsAdded, executorsRemoved, runAggregates),
   };
   // The one cast from the posted-model types to the detector-side shapes: types.ts's Stage and
   // SqlExecution carry a catch-all index signature, while every field DetectorStage/DetectorSqlExec
@@ -211,6 +232,7 @@ export function analyze(
     app, jobs, executorsAdded, executorsRemoved, runAggregates, impact,
     stages: stages                                    ,
     sql: sql                                 ,
+    skewThresholds: skewThresholdsFor(thresholds),
   };
   const out            = [];
   for (const d of detectors) {
@@ -238,11 +260,10 @@ export function analyze(
         assertNever(d);
     }
   }
-  const findings = applySuppression(out);
+  const findings = reconcileExecutorMemoryAdvice(applySuppression(out));
   estimateImpact(findings, impact);
   noteTunedThresholds(findings);
   deriveImpactBand(findings, app, thresholds ? (type) => bandFloors(type, thresholds) : undefined);
-  flagSkewStragglerOverlap(findings);
   // Ascending IMPACT_BAND_ORDER (critical 0 -> info 2) puts the worst band first;
   // stable sort keeps DETECTORS declaration order within a band.
   findings.sort((a, b) => IMPACT_BAND_ORDER[a.impactBand] - IMPACT_BAND_ORDER[b.impactBand]);
@@ -250,14 +271,17 @@ export function analyze(
 }
 
 // Memoizes auditConfig by `app` identity (like evidence-report.ts's jsonCache) so config detectors
-// don't re-run when both the export and the report path audit the same app. WeakMap can't key on
-// `null`, so that case skips the cache. Returns a fresh copy each call so a caller's in-place
-// mutation (e.g. `.sort()`) can't corrupt the cached array.
-const auditConfigCache = new WeakMap                         ();
+// don't re-run when both the export and the report path audit the same app. The one stage-dependent
+// check reads only whether any stage ran outside SQL, so that flag is part of the cache entry: a
+// hit with a different flag recomputes. WeakMap can't key on `null`, so that case skips the cache.
+// Returns a fresh copy each call so a caller's in-place mutation (e.g. `.sort()`) can't corrupt the
+// cached array.
+const auditConfigCache = new WeakMap                                                         ();
 
-function computeAuditConfig(app                     )            {
+function computeAuditConfig(app                     , stages                                )            {
   const out            = [];
-  for (const d of detectors) if (d.scope === 'config') push(out, d, d.withThresholds()({ app }));
+  const target = { app, stages: stages                                              };
+  for (const d of detectors) if (d.scope === 'config') push(out, d, d.withThresholds()(target));
   // configAudit's estimate is unconditionally costOnly('none'): needs no stages/totalCores, so an
   // empty context gives parity with analyze().
   estimateImpact(out, { stages: new Map(), occupancy: new Map(), totalCores: 0 });
@@ -265,11 +289,14 @@ function computeAuditConfig(app                     )            {
   return out.map((f) => ({ ...f, stageId: f.stageId ?? null }));
 }
 
-export function auditConfig(app                     )            {
-  if (app === null) return computeAuditConfig(app);
+/** The configuration findings for a run. `stages` lets checks that apply only to some kinds of
+ * work (the serializer note needs RDD stages) see the run's work; without it they stay silent. */
+export function auditConfig(app                     , stages                     )            {
+  if (app === null) return computeAuditConfig(app, stages);
+  const rddWork = hasStageOutsideSql(stages                                             );
   const cached = auditConfigCache.get(app);
-  if (cached) return cached.slice();
-  const result = computeAuditConfig(app);
-  auditConfigCache.set(app, result);
-  return result.slice();
+  if (cached?.rddWork === rddWork) return cached.findings.slice();
+  const findings = computeAuditConfig(app, stages);
+  auditConfigCache.set(app, { rddWork, findings });
+  return findings.slice();
 }

package/vendor-core/api-merge-writes.js

@@ -74,6 +74,8 @@ export function apiMergeWrites(sql                             )
       kind: resolved?.kind ?? null,
       target: resolved?.target ?? null,
       outputRows: null,
+      outputRowsSource: null,
+      outputBytes: null,
       mergeRows: null,
       raw: `Delta MERGE operation, executions ${span}${group.unclean ? ` (${group.unclean})` : ''}`,
     };