sparkforensics-server 0.5.1 → 0.6.0

package/public/favicon.svg

@@ -1,4 +1,4 @@
 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32">
-  <rect width="32" height="32" rx="7" fill="#181614"/>
-  <path d="M17.5 5 8 18h6.5l-1 9L24 14h-6.5l1-9z" fill="#ff6b2c"/>
+  <rect width="32" height="32" rx="6" fill="#4f46e5"/>
+  <path d="M17.5 5 8 18h6.5l-1 9L24 14h-6.5l1-9z" fill="#eceafd"/>
 </svg>

package/public/index.html

@@ -10,9 +10,6 @@
     <meta property="og:url" content="https://shuffle-works.github.io/sparkforensics/">
     <meta property="og:title" content="SparkForensics">
     <meta property="og:description" content="Zero-backend browser dashboard that parses Apache Spark event logs to diagnose performance bottlenecks: skew, spill, GC pressure, stragglers, cache utilization, and more.">
-    <link rel="preconnect" href="https://fonts.googleapis.com">
-    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-    <link href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
     <script>
       try {
         var stored = localStorage.getItem('shuffle-works-theme');
@@ -23,14 +20,14 @@
           document.documentElement.setAttribute('data-theme', 'light');
       } catch (_) {}
     </script>
-    <script type="module" crossorigin src="./assets/index-DCeWge12.js"></script>
+    <script type="module" crossorigin src="./assets/index-DwqkFUFk.js"></script>
     <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
     <link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
     <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-BYl-AzN5.js">
-    <link rel="modulepreload" crossorigin href="./assets/react-vendor-DEdfrRTJ.js">
-    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-B0siMVdg.js">
+    <link rel="modulepreload" crossorigin href="./assets/react-vendor-DfpuPQZf.js">
+    <link rel="modulepreload" crossorigin href="./assets/DisclosureContext-OO7X1HAx.js">
     <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-DLioOiRN.css">
-    <link rel="stylesheet" crossorigin href="./assets/index-BcqTLKFp.css">
+    <link rel="stylesheet" crossorigin href="./assets/index-DhpRGMZc.css">
   </head>
   <body>
     <div id="root"></div>

package/vendor-core/allocation.js

@@ -1,7 +1,9 @@
+import { computePeakConcurrentExecutorCount } from './core-count.js';
 import { parseSparkMemoryMB } from './spark-memory.js';
-                                                                                         
+                                                      
 
 const MS_PER_HOUR = 3_600_000;
+const MS_PER_SECOND = 1000;
 const MIB_PER_GIB = 1024;
 // Spark's documented floor and factor for the default executor memory overhead
 // (spark.executor.memoryOverhead = max(factor * executor memory, 384 MiB)).
@@ -14,12 +16,29 @@ const DEFAULT_OVERHEAD_FACTOR = 0.1;
                                                                                              
                                                         
                                
+                                                                                               
+                                                                                              
+                                         
+                                                                                     
+                               
+                                                                                               
+                                                                                                    
+                               
+                                                                                               
+                               
+                                                                                     
+                                 
  
 
+// The fields the allocation reads from the parsed model's executor events, so a detector's
+// narrower event shapes qualify too.
+                                                                                      
+                                                                   
+
                                   
-                           
-                             
+                                                                     
                                                                   
+                                                                    
  
 
 /** The latest timestamp the log records: the application start and end and every stage and
@@ -43,7 +62,7 @@ const DEFAULT_EXECUTOR_MEMORY_MIB = 1024;
 // plus overhead, plus off-heap and PySpark worker memory when configured. Null when the log
 // records no Spark properties (nothing to tell a default from a missing config) or a memory key it
 // does record cannot be read.
-function executorMemoryMiB(app                     )                {
+function executorMemoryMiB(app                        )                {
   const config = app?.config;
   if (config == null) return null;
   // undefined: key absent; null: present but unreadable.
@@ -63,6 +82,11 @@ function executorMemoryMiB(app                     )                {
   return heap + overhead + offHeap + pyspark;
 }
 
+function loggedDynamicAllocation(app                        )                      {
+  const raw = app?.config?.['spark.dynamicAllocation.enabled'];
+  return raw == null ? null : raw.trim().toLowerCase() === 'true' ? 'on' : 'off';
+}
+
 /** Allocated core-hours and memory GiB-hours from the executor lifecycle: each executor counts
  * from its ExecutorAdded timestamp to its first later ExecutorRemoved timestamp. One with no
  * removal closes at the application end when the log has one, else at the last timestamp the log
@@ -72,35 +96,96 @@ function executorMemoryMiB(app                     )                {
  * else the larger of 384 MiB and spark.executor.memoryOverheadFactor, default 0.1, times the
  * memory), plus spark.memory.offHeap.size when spark.memory.offHeap.enabled is true, plus
  * spark.executor.pyspark.memory.
+ * The executor counts and seconds come from the same alive intervals.
  * Null, never 0, for a figure whose inputs the log lacks. */
 export function computeAllocation(input                 )             {
-  const added = input.executors.added.filter((e)                          => e.kind === 'added');
-  if (added.length === 0) return { coreHours: null, memoryGbHours: null };
-  const removedAt = new Map                  ();
-  for (const e of input.executors.removed) {
-    if (e.kind !== 'removed') continue;
-    const times = removedAt.get(e.executorId);
-    if (times) times.push(e.timestamp); else removedAt.set(e.executorId, [e.timestamp]);
+  const dynamicAllocation = loggedDynamicAllocation(input.app);
+  const alive = aliveIntervals(input);
+  if (alive.length === 0) {
+    return { coreHours: null, memoryGbHours: null, dynamicAllocation, executorsPeak: null, executorsMean: null, executorCores: null, executorSeconds: null };
   }
   const closeAt = input.app?.endTime ?? lastObservedTimestamp(input);
-  const configuredCores = Number.parseInt(input.app?.config?.['spark.executor.cores'] ?? '', 10);
   const memoryMiB = executorMemoryMiB(input.app);
 
   let coreMs = 0;
   let memoryMiBMs = 0;
+  let aliveMsTotal = 0;
   let coresKnown = true;
-  const seen = new Set        ();
-  for (const e of added) {
-    if (seen.has(e.executorId)) continue; // a replayed ExecutorAdded is the same executor
-    seen.add(e.executorId);
-    const removal = (removedAt.get(e.executorId) ?? []).filter((t) => t >= e.timestamp).sort((a, b) => a - b)[0];
-    const aliveMs = Math.max(0, (removal ?? closeAt ?? e.timestamp) - e.timestamp);
-    const cores = e.totalCores > 0 ? e.totalCores : Number.isFinite(configuredCores) ? configuredCores : null;
-    if (cores == null) coresKnown = false; else coreMs += cores * aliveMs;
+  const coreCounts = new Set        ();
+  for (const { start, end, cores } of alive) {
+    const aliveMs = end - start;
+    aliveMsTotal += aliveMs;
+    if (cores == null) coresKnown = false; else { coreMs += cores * aliveMs; coreCounts.add(cores); }
     memoryMiBMs += (memoryMiB ?? 0) * aliveMs;
   }
+  const startTime = input.app?.startTime;
+  const windowMs = closeAt != null && startTime != null ? closeAt - startTime : 0;
+  const firstRemovals = alive.flatMap((a) => (a.removedAt == null ? [] : [{ executorId: a.executorId, timestamp: a.removedAt }]));
   return {
     coreHours: coresKnown ? coreMs / MS_PER_HOUR : null,
     memoryGbHours: memoryMiB != null ? memoryMiBMs / MIB_PER_GIB / MS_PER_HOUR : null,
+    dynamicAllocation,
+    executorsPeak: computePeakConcurrentExecutorCount(alive.map((a) => ({ executorId: a.executorId, timestamp: a.start })), firstRemovals),
+    executorsMean: windowMs > 0 ? aliveMsTotal / windowMs : null,
+    executorCores: coresKnown && coreCounts.size === 1 ? [...coreCounts][0] : null,
+    executorSeconds: aliveMsTotal / MS_PER_SECOND,
   };
 }
+
+                         
+                     
+                
+              
+                           
+                                                                                         
+                       
+ 
+
+// Each executor's alive interval: from its ExecutorAdded timestamp to its first later
+// ExecutorRemoved timestamp, else the application end, else the last timestamp the log records.
+// The one definition computeAllocation and allocatedCoreMsWithin share.
+function aliveIntervals(input                 )                  {
+  const removedAt = new Map                  ();
+  for (const e of input.executors.removed) {
+    const times = removedAt.get(e.executorId);
+    if (times) times.push(e.timestamp); else removedAt.set(e.executorId, [e.timestamp]);
+  }
+  const closeAt = input.app?.endTime ?? lastObservedTimestamp(input);
+  const configuredCores = Number.parseInt(input.app?.config?.['spark.executor.cores'] ?? '', 10);
+  const out                  = [];
+  const seen = new Set        ();
+  for (const e of input.executors.added) {
+    if (seen.has(e.executorId)) continue; // a replayed ExecutorAdded is the same executor
+    seen.add(e.executorId);
+    const removal = (removedAt.get(e.executorId) ?? []).filter((t) => t >= e.timestamp).sort((a, b) => a - b)[0] ?? null;
+    out.push({
+      executorId: e.executorId,
+      start: e.timestamp,
+      end: Math.max(e.timestamp, removal ?? closeAt ?? e.timestamp),
+      removedAt: removal,
+      cores: e.totalCores != null && e.totalCores > 0 ? e.totalCores : Number.isFinite(configuredCores) ? configuredCores : null,
+    });
+  }
+  return out;
+}
+
+/** Allocated core-milliseconds: the figure behind `coreHours`, the capacity idle-core figures are
+ * measured against. Null when the log has no executor events or an executor's cores are unknown. */
+export function allocatedCoreMs(input                 )                {
+  const { coreHours } = computeAllocation(input);
+  return coreHours != null && coreHours > 0 ? coreHours * MS_PER_HOUR : null;
+}
+
+/** The part of the allocated core-milliseconds held inside `windows` (disjoint, sorted
+ * intervals, such as the union of the stages' windows), counted from the same alive intervals as
+ * `allocatedCoreMs`. Null when `allocatedCoreMs` is. */
+export function allocatedCoreMsWithin(input                 , windows                         )                {
+  if (allocatedCoreMs(input) == null) return null;
+  let coreMs = 0;
+  for (const { start, end, cores } of aliveIntervals(input)) {
+    for (const [from, to] of windows) {
+      coreMs += (cores ?? 0) * Math.max(0, Math.min(end, to) - Math.max(start, from));
+    }
+  }
+  return coreMs;
+}

package/vendor-core/api-merge-writes.js

@@ -74,6 +74,7 @@ export function apiMergeWrites(sql                             )
       kind: resolved?.kind ?? null,
       target: resolved?.target ?? null,
       outputRows: null,
+      mergeRows: null,
       raw: `Delta MERGE operation, executions ${span}${group.unclean ? ` (${group.unclean})` : ''}`,
     };
   });

package/vendor-core/cli/budgets.js

@@ -3,6 +3,7 @@ import { computeSkewRatio, ENTRY_BY_TYPE,                         } from '../det
 import { effectiveThresholds } from '../threshold-overrides.js';
 import { IMPACT_BAND_ORDER } from '../format-utils.js';
                                                               
+import { PAIR_DELTA_METRICS,                                      } from '../stage-alignment.js';
                                                                  
                                                                 
 
@@ -14,13 +15,23 @@ const IMPACT_BANDS = Object.keys(IMPACT_BAND_ORDER)                ;
                                                                                   
                                                                                                  
                                          
+                                                                                                                        
+                                              
+                                                                                                            
+                                                                                   
+                                          
  
+
+export const STAGE_QUALITIES                                  = ['exact', 'structural', 'aligned'];
+export const DEFAULT_STAGE_BUDGET_QUALITIES                                  = ['exact', 'structural'];
+/** Metrics with a regression direction (higher is worse). Input and output volume are workload, not performance. */
+export const NEUTRAL_PAIR_METRICS                      = new Set(['inputBytes', 'outputBytes']);
                                
                                                                                             
-                                                               
+                                                                                        
                                                 
                  
-                                                                            
+                                                                                                       
                   
  
 
@@ -146,6 +157,50 @@ function checkRegression(comparison                   , maxRegressionPct
     : { name: 'max-regression', metric: regressionMetric, status: 'pass', detail: `Metric "${regressionMetric}" ${pctLabel}, within budget ${maxRegressionPct}%.` };
 }
 
+function checkStageRegression(
+  comparison                   , { metric, maxPct }                  , qualities                                 ,
+)               {
+  const name = 'max-stage-regression'         ;
+  if (!(PAIR_DELTA_METRICS                     ).includes(metric)) {
+    return { name, metric, status: 'inconclusive', detail: `Metric "${metric}" is not a paired-stage metric (expected one of: ${PAIR_DELTA_METRICS.join(', ')}).` };
+  }
+  if (NEUTRAL_PAIR_METRICS.has(metric)) {
+    return { name, metric, status: 'inconclusive', detail: `Metric "${metric}" measures workload volume, not performance: it has no regression direction to check.` };
+  }
+  const eligible = comparison.stagePairs.filter((p) => qualities.includes(p.quality));
+  if (eligible.length === 0) {
+    return { name, metric, status: 'inconclusive', detail: comparison.stagePairs.length === 0
+      ? 'No stages were paired between the two runs.'
+      : `None of the ${comparison.stagePairs.length} paired stage(s) has quality ${qualities.join(' or ')}.` };
+  }
+  const measured = eligible
+    .map((pair) => ({ pair, delta: pair.deltas[metric                   ] }))
+    .filter(({ delta }) => delta.baseline != null && delta.delta != null);
+  if (measured.length === 0) {
+    return { name, metric, status: 'inconclusive', detail: `Metric "${metric}" is unavailable for every eligible paired stage.` };
+  }
+  const regressions = measured
+    .filter(({ delta }) => delta.delta  > 0)
+    .map(({ pair, delta }) => ({ pair, delta, pct: delta.baseline === 0 ? Infinity : (delta.delta  / delta.baseline ) * 100 }))
+    .sort((a, b) => b.pct - a.pct || a.pair.pairId.localeCompare(b.pair.pairId));
+  const over = regressions.filter((r) => r.pct > maxPct);
+  const scope = `${measured.length} paired stage(s)`;
+  if (over.length === 0) {
+    const worst = regressions[0];
+    return { name, metric, status: 'pass', detail: worst
+      ? `No paired stage regressed "${metric}" beyond ${maxPct}% across ${scope}; the largest was ${formatPct(worst.pct)} (pair ${worst.pair.pairId}).`
+      : `No paired stage regressed "${metric}" across ${scope}.` };
+  }
+  const listed = over.slice(0, 3).map((r) => `pair ${r.pair.pairId} (${r.pair.quality}) ${formatPct(r.pct)}`).join(', ');
+  const more = over.length > 3 ? `, and ${over.length - 3} more` : '';
+  return { name, metric, status: 'violation',
+    detail: `${over.length} of ${scope} regressed "${metric}" beyond ${maxPct}%: ${listed}${more}.` };
+}
+
+function formatPct(pct        )         {
+  return pct === Infinity ? 'from 0' : `+${pct.toFixed(1)}%`;
+}
+
 function checkFailOnIntroduced(comparison                   , band        )               {
   if (band !== 'all' && !IMPACT_BANDS.includes(band              )) {
     return { name: 'fail-on-introduced', status: 'inconclusive', detail: `Impact band "${band}" is not recognized (expected "all" or one of ${IMPACT_BANDS.join(', ')}).` };
@@ -198,6 +253,11 @@ export function evaluateBudgets({ appModel, catalog, budgets, comparison, thresh
   for (const { metric, maxPct } of budgets.regressionBudgets ?? []) {
     pushComparisonBudget(results, comparison, 'max-regression', (c) => checkRegression(c, maxPct, metric), metric);
   }
+  const stageQualities = budgets.stageQualities ?? DEFAULT_STAGE_BUDGET_QUALITIES;
+  for (const budget of budgets.stageRegressionBudgets ?? []) {
+    pushComparisonBudget(results, comparison, 'max-stage-regression',
+      (c) => checkStageRegression(c, budget, stageQualities), budget.metric);
+  }
   if (budgets.failOnIntroduced !== undefined) {
     pushComparisonBudget(results, comparison, 'fail-on-introduced',
       (c) => checkFailOnIntroduced(c, budgets.failOnIntroduced ));

package/vendor-core/cli/regression-budgets.js

@@ -4,6 +4,9 @@
 import { readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { COMPARISON_METRIC_KEYS } from '../run-comparison.js';
+import { PAIR_DELTA_METRICS } from '../stage-alignment.js';
+import { NEUTRAL_PAIR_METRICS, STAGE_QUALITIES } from './budgets.js';
+                                                       
 
                                                                      
                                                                                      
@@ -81,3 +84,35 @@ export function combineRegressionBudgets(sources                          )
   }
   return sources.map((s) => s.budget);
 }
+
+/** Refuses a stage budget metric that is not a paired-stage metric or has no regression direction. */
+export function assertStageBudgetMetric(metric        , origin        )       {
+  if (!(PAIR_DELTA_METRICS                     ).includes(metric)) {
+    throw new Error(`${origin}: unknown paired-stage metric "${metric}" (expected one of: ${PAIR_DELTA_METRICS.filter((m) => !NEUTRAL_PAIR_METRICS.has(m)).join(', ')}).`);
+  }
+  if (NEUTRAL_PAIR_METRICS.has(metric)) {
+    throw new Error(`${origin}: "${metric}" measures workload volume, not performance, so it has no regression direction to budget.`);
+  }
+}
+
+/** One `--stage-regression-budget` value, `<metric>:<pct>`, over the paired-stage metrics. */
+export function parseStageRegressionBudgetFlag(spec        )                   {
+  const origin = `--stage-regression-budget "${spec}"`;
+  const colon = spec.indexOf(':');
+  if (colon === -1) throw new Error(`${origin}: expected <metric>:<pct>.`);
+  const metric = spec.slice(0, colon);
+  const pct = spec.slice(colon + 1);
+  assertStageBudgetMetric(metric, origin);
+  if (!PCT_PATTERN.test(pct)) throw new Error(`${origin}: "${pct}" is not a non-negative percentage.`);
+  return { metric, maxPct: Number(pct) };
+}
+
+/** The pair qualities a stage budget reads, from a comma list such as `exact,structural`. */
+export function parseStageQualities(list        , origin        )                         {
+  const qualities = list.split(',').map((q) => q.trim()).filter(Boolean);
+  const bad = qualities.find((q) => !(STAGE_QUALITIES                     ).includes(q));
+  if (qualities.length === 0 || bad !== undefined) {
+    throw new Error(`${origin}: ${bad === undefined ? 'no quality given' : `unknown quality "${bad}"`} (expected a list from: ${STAGE_QUALITIES.join(', ')}).`);
+  }
+  return qualities                          ;
+}

package/vendor-core/comparison-output.js

@@ -11,7 +11,7 @@ import { redactComparison } from './redact.js';
                                                                  
                            
                                                                                             
-                                                                                    
+                                                                                                           
                                                    
 
 /** `full` carries every field (the CLI); `summary` leaves out `stagePairs` (MCP, which returns them
@@ -33,6 +33,7 @@ export function comparisonOutput(comparison                   , view
     unmatched: comparison.unmatched,
     replanned: comparison.replanned,
     bookkeepingStageIds: comparison.bookkeepingStageIds,
+    executionAlignment: comparison.executionAlignment,
   };
 }
 

package/vendor-core/core-source-hash.txt

@@ -1 +1 @@
-6e0ac29f74ae092b14c5700dfa86d3273fce7d1af59005c6e2e450a91da92d5e
+9c33274bcae34f425c7d625bdea9cfacdb400f333bc3a5ced9c86a0e75094916

package/vendor-core/delta-log-paths.js

@@ -9,6 +9,15 @@ import { isCut,                  } from './write-target-args.js';
 // scans and `DeltaLogFileIndex [<path>/_delta_log/<file>, ...]`.
 const DELTA_LOG_PATH = /((?:[A-Za-z][A-Za-z0-9+.-]*:\/{1,3}|\/)[^\s,[\]()#]+?)\/_delta_log(?=[/[\],\s)]|$)/g;
 
+// Hadoop prints a local path as `file:/p` and a URI as `file:///p`: one table, two spellings.
+// Both read as `file:///p` so a table is not counted twice. A `file://host/p` is left alone.
+const LOCAL_FILE_PREFIX = /^file:(?:\/|\/\/\/)(?=[^/])/;
+
+/** The one spelling of a table path: the two local-file forms agree, other paths pass through. */
+export function normalizeTablePath(path        )         {
+  return path.replace(LOCAL_FILE_PREFIX, 'file:///').replace(/\/+$/, '');
+}
+
                                                               
 
 // Every distinct table path named in the plans; `unreliable` when a plan is missing or a path cut.
@@ -26,7 +35,7 @@ function deltaLogPaths(executions                                  )           {
         if (matches.length < mentions) unreliable = true;
         for (const m of matches) {
           if (isCut(m[1])) unreliable = true;
-          else paths.add(m[1].replace(/\/+$/, ''));
+          else paths.add(normalizeTablePath(m[1]));
         }
       }
     }, { dedupe: true });