sparkforensics-server 0.5.2 → 0.8.1

package/vendor-core/stage-quantiles.js

@@ -1,6 +1,7 @@
                                                             
 import { medianOfSorted } from './median.js';
                                                                              
+                                                  
 
 // Single source of truth for the packed per-task numeric array: FIELDS (offset
 // constants), TASK_FIELD_NAMES (display labels), and the finalizeStage hot-loop
@@ -43,9 +44,8 @@ export const TASK_FIELD_NAMES           = TASK_FIELD_DESCRIPTORS.map((d) => d.na
 const TASK_FIELD_PROPS = TASK_FIELD_DESCRIPTORS.map((d) => d.prop);
 
 // Numeric accumulator fields on `stage` that this function reads/increments.
-// `stage`'s public parameter type stays the loose Record shape per this
-// module's Produces contract; this narrow view is only for the arithmetic
-// below, never a restatement of the object's real shape.
+// `stage`'s public parameter type stays the loose Record shape; this narrow view is only for the
+// arithmetic below, never a restatement of the object's real shape.
                                    
                            
                             
@@ -82,7 +82,8 @@ export function finalizeStage(
   const localityStats = new Map();
   let peakExecutionMemoryMax = 0;
 
-  for (const t of stage.taskAttempts.values()) {
+  const attempts = stage.taskAttempts;
+  for (const t of attempts.values()) {
     taskCount++;
     if (t.failed) {
       failedTasks++;
@@ -105,7 +106,15 @@ export function finalizeStage(
     }
     if (t.executorId) {
       let es = executorStats.get(t.executorId);
-      if (!es) { es = { taskCount: 0, totalDuration: 0, inputBytes: 0, shuffleReadBytes: 0, shuffleWriteBytes: 0 }; executorStats.set(t.executorId, es); }
+      if (!es) { es = { taskCount: 0, totalDuration: 0, logDurationSum: 0, logDurationM2: 0, inputBytes: 0, shuffleReadBytes: 0, shuffleWriteBytes: 0 }; executorStats.set(t.executorId, es); }
+      // Task durations scatter multiplicatively, so the spread slowHost tests an executor against is
+      // kept on the log scale: a running sum and Welford's M2 of ln(1 + duration). A non-finite
+      // duration makes both NaN, which the detector reads as "no spread known".
+      const logDuration = Math.log1p(t.duration);
+      const logMeanBefore = es.taskCount > 0 ? es.logDurationSum / es.taskCount : 0;
+      const logMeanAfter = (es.logDurationSum + logDuration) / (es.taskCount + 1);
+      es.logDurationM2 += (logDuration - logMeanBefore) * (logDuration - logMeanAfter);
+      es.logDurationSum += logDuration;
       es.taskCount++;
       es.totalDuration += t.duration;
       es.inputBytes += t.inputBytes;
@@ -145,9 +154,9 @@ export function finalizeStage(
   const { p50: spillMemP50, p95: spillMemP95, max: spillMemMax } = computeFieldQuantiles(arr, FIELDS.MEM_SPILLED);
   const { p50: spillDiskP50, p95: spillDiskP95, max: spillDiskMax } = computeFieldQuantiles(arr, FIELDS.DISK_SPILLED);
 
-  // Straggler count: tasks with duration > 4 * P50, their summed excess over P50, and the longest
-  // task that isn't one.
-  const stragglerThreshold = 4 * p50;
+  // Straggler count: tasks with duration > STRAGGLER_FACTOR * P50, their summed excess over P50, and
+  // the longest task that isn't one.
+  const stragglerThreshold = STRAGGLER_FACTOR * p50;
   let stragglerCount = 0;
   let stragglerExcessMs = 0;
   let longestNonStragglerMs = 0;
@@ -160,6 +169,7 @@ export function finalizeStage(
     }
   }
 
+  const tailAttribution = p50 > 0 ? attributeTail(attempts.values()                                 , p50) : null;
   const peakConcurrentTasks = computePeakConcurrentTasks(arr);
   const tailReplayRecoveryMs = stragglerCount > 0
     ? computeTailReplayRecoveryMs(arr, p50, peakConcurrentTasks)
@@ -169,7 +179,7 @@ export function finalizeStage(
     ([host, s]) => ({ host, taskCount: s.taskCount, totalDuration: s.totalDuration })
   );
   const executorStatsArr = [...executorStats.entries()].map(
-    ([executorId, s]) => ({ executorId, taskCount: s.taskCount, totalDuration: s.totalDuration, inputBytes: s.inputBytes, shuffleReadBytes: s.shuffleReadBytes, shuffleWriteBytes: s.shuffleWriteBytes })
+    ([executorId, s]) => ({ executorId, taskCount: s.taskCount, totalDuration: s.totalDuration, logDurationSum: s.logDurationSum, logDurationM2: s.logDurationM2, inputBytes: s.inputBytes, shuffleReadBytes: s.shuffleReadBytes, shuffleWriteBytes: s.shuffleWriteBytes })
   );
   const failureReasonsArr = [...failureReasons.entries()].map(
     ([reason, count]) => ({ reason, count })
@@ -191,6 +201,7 @@ export function finalizeStage(
     peakExecutionMemoryMax,
     taskActiveMs: computeTaskActiveMs(arr),
     peakConcurrentTasks,
+    ...(tailAttribution ? { tailAttribution } : {}),
     taskDurationP50: p50,
     taskDurationP95: p95,
     taskDurationMax: max,
@@ -210,6 +221,127 @@ export function finalizeStage(
   return { type: 'stage', data };
 }
 
+// A task is in the tail when it runs over this many times the stage's median task: skew's own
+// P95/median ratio threshold, so every stage either detector flags on duration has a tail here.
+export const TAIL_FACTOR = 3;
+
+// A task is a straggler when it runs over this many times the stage's median task.
+export const STRAGGLER_FACTOR = 4;
+
+// The per-task fields attributeTail reads; a TaskRecord in event-handlers.ts has all of them.
+                    
+                  
+                   
+               
+                 
+                        
+                          
+                          
+                     
+                       
+                      
+                             
+ 
+
+// A hosts-concentration reading needs this many tail tasks on the host, at least this share of the
+// tail, and at least this multiple of the host's share of all tasks, before the host explains the
+// excess the other causes leave. One slow task on a host that ran a tenth of the stage is chance.
+const TAIL_HOST_MIN_TASKS = 3;
+const TAIL_HOST_MIN_TAIL_SHARE = 0.5;
+const TAIL_HOST_MIN_OVERREPRESENTATION = 2;
+
+// A median task that read next to nothing makes every ratio against it huge (a median of 4 bytes
+// against a 1 MB task reads as 250000x). Below these floors the median task is read as having
+// read almost nothing: there is no ratio to take, and a task counts as having read data only from
+// the floors up.
+const MIN_VOLUME_BYTES = 1024 * 1024;
+const MIN_VOLUME_RECORDS = 1000;
+
+// Why the stage's slow tasks (non-failed, over TAIL_FACTOR x P50) were slow, from what each task
+// logged. A task's excess is its duration over P50. Four causes claim it, in this order so no
+// millisecond counts twice:
+//   1. Data: had run time scaled with data volume, the task would have taken P50 x (volume ratio)
+//      against the median task, with volume the task's input plus shuffle-read, in bytes or in
+//      records, whichever is further above the median. This is the definition Spark's UI and AQE
+//      use for skew (a partition far larger than the median one), applied to run time.
+//   2. GC time over the median task's.
+//   3. Shuffle fetch wait over the median task's.
+//   4. Host: what is left of the tail tasks on the one host that holds most of the tail.
+// With a median task that read almost nothing there is no ratio to take, so GC and fetch wait claim their
+// excess first and a task that read data owns what they leave: the time was GC or waiting unless
+// the data is all that is left to explain it.
+// Null when the stage has no tail. `tasks` is consumed once.
+export function attributeTail(tasks                    , p50        )                         {
+  const all             = [];
+  const tail             = [];
+  for (const t of tasks) {
+    if (t.failed) continue; // a failed attempt's metrics stop where it died
+    all.push(t);
+    if (t.duration > TAIL_FACTOR * p50) tail.push(t);
+  }
+  if (tail.length === 0) return null;
+
+  const bytesOf = (t          ) => t.inputBytes + t.shuffleRead;
+  const recordsOf = (t          ) => t.inputRecords + t.shuffleReadRecords;
+  const medianBytes = medianOf(all, bytesOf);
+  const medianRecords = medianOf(all, recordsOf);
+  const medianGc = medianOf(all, (t) => t.gcTime);
+  const medianFetch = medianOf(all, (t) => t.fetchWaitTime);
+  const bytesComparable = medianBytes >= MIN_VOLUME_BYTES;
+  const recordsComparable = medianRecords >= MIN_VOLUME_RECORDS;
+  const hasVolume = bytesComparable || recordsComparable;
+  const volumeRatio = (t          )         => Math.max(
+    bytesComparable ? bytesOf(t) / medianBytes : 0,
+    recordsComparable ? recordsOf(t) / medianRecords : 0,
+  );
+  const readsData = (t          ) => bytesOf(t) >= MIN_VOLUME_BYTES || recordsOf(t) >= MIN_VOLUME_RECORDS;
+
+  const tailHosts = new Map                ();
+  const allHosts = new Map                ();
+  for (const t of all) if (t.host) allHosts.set(t.host, (allHosts.get(t.host) ?? 0) + 1);
+  for (const t of tail) if (t.host) tailHosts.set(t.host, (tailHosts.get(t.host) ?? 0) + 1);
+  let topHost                = null;
+  for (const [host, n] of tailHosts) if (topHost === null || n > tailHosts.get(topHost) ) topHost = host;
+  const hostConcentrated = topHost !== null
+    && tailHosts.get(topHost)  >= TAIL_HOST_MIN_TASKS
+    && tailHosts.get(topHost)  / tail.length >= TAIL_HOST_MIN_TAIL_SHARE
+    && (tailHosts.get(topHost)  / tail.length) / (allHosts.get(topHost)  / all.length) >= TAIL_HOST_MIN_OVERREPRESENTATION;
+
+  let excessMs = 0, dataMs = 0, gcMs = 0, fetchWaitMs = 0, hostMs = 0, cpuNs = 0, runMs = 0;
+  const ratios           = [];
+  for (const t of tail) {
+    const excess = t.duration - p50;
+    excessMs += excess;
+    let left = excess;
+    if (hasVolume) {
+      const ratio = volumeRatio(t);
+      ratios.push(ratio);
+      const data = Math.min(left, Math.max(0, p50 * (ratio - 1)));
+      dataMs += data; left -= data;
+    }
+    const gc = Math.min(left, Math.max(0, t.gcTime - medianGc));
+    gcMs += gc; left -= gc;
+    const fetch = Math.min(left, Math.max(0, t.fetchWaitTime - medianFetch));
+    fetchWaitMs += fetch; left -= fetch;
+    if (!hasVolume && readsData(t)) { dataMs += left; left = 0; }
+    if (hostConcentrated && t.host === topHost) { hostMs += left; }
+    cpuNs += t.executorCpuTime;
+    runMs += t.executorRunTime;
+  }
+  ratios.sort((a, b) => a - b);
+  return {
+    tasks: tail.length, excessMs, dataMs, gcMs, fetchWaitMs, hostMs,
+    host: hostConcentrated ? topHost : null,
+    hostTasks: hostConcentrated ? tailHosts.get(topHost )  : 0,
+    dataRatio: hasVolume ? medianOfSorted(Float64Array.from(ratios)) : null,
+    cpuPct: cpuNs > 0 && runMs > 0 ? Math.min(100, (cpuNs / 1e6 / runMs) * 100) : null,
+  };
+}
+
+function medianOf(tasks            , valueOf                         )         {
+  return medianOfSorted(Float64Array.from(tasks, valueOf).sort());
+}
+
 // Wall-clock time during which at least one of the stage's tasks was running: the union of its
 // [launch, finish) intervals. A stage's submittedAt..completedAt window also covers time it sat
 // open with no task running (waiting for a free slot, or between retried tasks), which no

package/vendor-core/stage-structure.js

@@ -3,7 +3,7 @@
 // removed) and the token sets its similarity scores read. Pure.
 
 import { cyrb53 } from './string-hash.js';
-import { normalizeStageName,                       } from './stage-identity.js';
+import { normalizeStageName, normalizedDetailOf,                       } from './stage-identity.js';
                                            
 
 // Plumbing nodes: codegen boundaries and input adapters change with how Spark fuses a stage, not with
@@ -43,7 +43,7 @@ export function structuralKey(nodes                     )                {
   const digestOf = (node          )         => {
     let d = digests.get(node);
     if (d === undefined) {
-      d = cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), attributeNames(node.detail ?? ''), below(node).sort()]));
+      d = cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), attributeNamesOf(node), below(node).sort()]));
       digests.set(node, d);
     }
     return d;
@@ -53,6 +53,20 @@ export function structuralKey(nodes                     )                {
   return cyrb53(JSON.stringify(roots));
 }
 
+// A node's attribute names are read by the structural key of every stage it ran in and by the
+// execution's plan structure; a node is never mutated after the parser posts it.
+const attributeNamesCache = new WeakMap                    ();
+
+/** `attributeNames` of a plan node's detail, memoized per node. The array is shared: do not change it. */
+export function attributeNamesOf(node          )                    {
+  let names = attributeNamesCache.get(node);
+  if (names === undefined) {
+    names = attributeNames(node.detail ?? '');
+    attributeNamesCache.set(node, names);
+  }
+  return names;
+}
+
 const TOKEN_SPLIT = /[^A-Za-z0-9_.$]+/;
 
 /** Sorted distinct tokens of a stage's attributed nodes (name and normalized detail) and its name.
@@ -64,7 +78,7 @@ export function detailTokens(stageName                    , nodes
   for (const node of nodes) {
     if (isPlumbing(node)) continue;
     add(normalizeStageName(node.name ?? ''));
-    add(normalize(node.detail ?? ''));
+    add(normalizedDetailOf(node, normalize));
   }
   return [...tokens].sort();
 }

package/vendor-core/threshold-overrides.js

@@ -88,11 +88,17 @@ export function tunedThresholdsOf(entry          , overrides
   return Object.keys(tuned).length > 0 ? tuned : null;
 }
 
-/** The tuned thresholds an entry's findings carry: its own, plus its `suppressedBy` entry's
- * (named `<suppressor>.<threshold>`), since tuning the suppressor changes which of them survive. */
+// straggler judges a tail with skew's resolved thresholds (DetectorCtx.skewThresholds), so tuning skew
+// changes which tails it reports without any `suppressedBy` link.
+const THRESHOLD_DEPENDENCY                                   = { straggler: 'skew' };
+
+/** The tuned thresholds an entry's findings carry: its own, plus its `suppressedBy` entry's or the
+ * entry it reads thresholds from (named `<entry>.<threshold>`), since tuning either changes which
+ * of them survive. */
 export function findingTunedThresholds(entry          , overrides                                )                         {
   const own = tunedThresholdsOf(entry, overrides);
-  const suppressor = entry.suppressedBy ? entries.find((d) => d.type === entry.suppressedBy) : undefined;
+  const suppressorType = entry.suppressedBy ?? THRESHOLD_DEPENDENCY[entry.type];
+  const suppressor = suppressorType ? entries.find((d) => d.type === suppressorType) : undefined;
   const bySuppressor = suppressor ? tunedThresholdsOf(suppressor, overrides) : null;
   if (!suppressor || !bySuppressor) return own;
   const prefixed = Object.fromEntries(Object.entries(bySuppressor).map(([name, t]) => [`${suppressor.type}.${name}`, t]));

package/vendor-core/types.js

@@ -43,8 +43,7 @@
                                                                              
 // Contract-pinned evidence `eventType`: the same fixed set as the parser's
 // `evidenceInputs` counter keys, so provenance stays addressable against the
-// documented `{ eventType, count }` shape (review: values must not diverge
-// from the counters they summarize).
+// documented `{ eventType, count }` shape.
                                
                                                                                          
                                                                                 
@@ -88,10 +87,10 @@
                        
                       
                        
-                                                                                                  
+                                                                                                                         
                                 
                                                                                         
-                                                                    
+               
                          
                                
                                                                                                
@@ -120,7 +119,7 @@
                            
                                   
                                                                                       
-                                                                           
+                          
                         
                                                                                         
                              
@@ -131,20 +130,51 @@
                                 
                                                                                             
                                
+                                                                                                
+                     
+                                    
                                                         
                    
                          
+ 
+
+// The tail of a stage: its non-failed tasks over TAIL_FACTOR x P50. Each cause is the summed time
+// (ms) of the tail's excess over P50 that the cause accounts for, taken in the order data, GC, fetch
+// wait, host so that no millisecond counts twice; what none of them accounts for is
+// excessMs - dataMs - gcMs - fetchWaitMs - hostMs.
+                                  
+                
+                   
+                                                                                                
+                                                                               
+                 
+               
+                      
+                                                                                                
+                                                                                                 
+                 
+                      
+                                                         
+                    
+                                                                                                    
+                                                                                                 
+                                                                      
+                           
+                                                                                                      
+                        
  
 
                            
                                                                              
-                                                                            
-                                                                        
-                                                                           
+                                                                               
+                                                                     
               
                
                   
-                                                                   
+                                                                                                
+                                                                                              
+                                                                                        
+                                                                                        
                        
                       
                                                                     
@@ -161,6 +191,8 @@
                            
                                                                                                         
                             
+                                                                                                                            
+                                           
                          
  
                                     
@@ -179,19 +211,14 @@
                                     
                                
                                                                            
-                                                                 
-                                                             
-                                                                          
-                                                                          
-                                                                        
-                                                                        
-                                                                      
-                                                                           
-                                                                           
-                                                                
+                                                                            
+                                                                       
+                                                                               
+                                                                                
+                                                                                
                                          
                                                                             
-                                                            
+                                    
                        
                                                                           
                                                                           
@@ -289,6 +316,9 @@
                       
                                
                                                                                  
+                                                                                               
+                                                                                            
+                                                               
                          
  
 
@@ -316,6 +346,9 @@
                                                                                          
                                                                                     
               
+                                                                                       
+                                               
+                
  
 
                                                                                             
@@ -332,15 +365,29 @@
                                                   
                                        
                             
+                                                                   
+                                                                                               
+                                                                                               
+                       
                                                                                                
                                                                                                
                                                                                              
                                                                                                  
                                                                                                
                                                                                    
-                                                                                                                                  
-                                                                     
+                                                                                                  
+                                                                                             
+                                                                                                 
+                                                                                                  
+                        
                                                     
+                                                                                                 
+                                                                                                  
+                                                                                                
+                                                                                                    
+                                                                                                  
+                                                                                        
+                                                 
  
 
 // `Finding` is a union discriminated on `type`, one member per emitted finding type: see
@@ -353,7 +400,7 @@
                 
                  
                          
-                                                                             
+                                                                         
                                                                          
                                                                            
                    

package/vendor-core/wall-clock.js

@@ -1,5 +1,13 @@
 import { mergeIntervals } from './intervals.js';
 
+/** The union of the stages' [submittedAt, completedAt) windows, sorted: where stages were active. */
+export function stageWindows(stages                                                           )                          {
+  const intervals = [...stages.values()]
+    .filter((s) => s.submittedAt != null && s.completedAt != null)
+    .map((s)                   => [s.submittedAt          , s.completedAt          ]);
+  return mergeIntervals(intervals);
+}
+
 export function computeWallClock(app                                                             , stages                                                           )                                                                                     {
   const start = app?.startTime ?? 0;
   const stageList = [...stages.values()].filter(s => s.submittedAt != null && s.completedAt != null);

package/vendor-core/wasted-core-hours.js

@@ -1,7 +1,8 @@
 // Wasted core-hours summary. Pure, worker-agnostic reducer over the same inputs
-// the memoryUtilization detector's idle-cores math uses: capacity core-time vs.
+// the memoryUtilization detector's idle-cores math uses: allocated core-time vs.
 // core-time that actually ran tasks. Feeds a report widget only, no detector.
 
+import { allocatedCoreMs } from './allocation.js';
 import { computePeakConcurrentCores } from './core-count.js';
 import { MS_PER_CORE_HOUR } from './format-utils.js';
 
@@ -32,29 +33,28 @@ const EMPTY                        = {
 // SparkAppInfo | null before parsing completes), which the guard below already
 // tolerates. `resources` is on app's shape so the computePeakConcurrentCores
 // pass-through type-checks; real app objects always carry it. An omitted
-// `executorsRemoved` means no executor left, where peak cores equal the sum.
+// `executorsRemoved` means no executor left.
 export function computeWastedCoreHours(
-  app                                                                                                       ,
+  app                                                                                                                                        ,
   executorsAdded                                                                                    = [],
   runAggregates                      ,
   executorsRemoved                                                   = [],
 )                        {
-  // Nullish (not falsy) guard on times: a literal startTime:0 is valid.
   if (!runAggregates || app?.startTime == null || app?.endTime == null) return EMPTY;
   const appDurationMs = app.endTime - app.startTime;
   if (appDurationMs <= 0) return EMPTY;
 
-  // Total cores: peak concurrent Executor-Added Total Cores, else peak executors ×
-  // configured cores. Same capacity as the memoryUtilization detector, so an
-  // executor replaced mid-run is not counted twice.
+  // Capacity is the run's allocation (cores x time alive), the same as the idle findings. The
+  // application end is set here, so the allocation never needs the stages to find where a log stops.
+  const allocatedMs = allocatedCoreMs({ app, stages: new Map(), executors: { added: executorsAdded, removed: executorsRemoved } });
+  if (allocatedMs == null) return EMPTY;
+  // Peak concurrent cores, reported as the cluster size: an executor replaced mid-run counts once.
   const totalCores = computePeakConcurrentCores(app, executorsAdded, executorsRemoved);
-  if (totalCores <= 0) return EMPTY;
 
-  const totalCoreHours = (totalCores * appDurationMs) / MS_PER_CORE_HOUR;
+  const totalCoreHours = allocatedMs / MS_PER_CORE_HOUR;
   const usefulCoreHours = (runAggregates.busyCoreMs ?? 0) / MS_PER_CORE_HOUR;
-  // Clamp: totalCores can be a fallback estimate while busyCoreMs is measured
-  // from task events, so the difference can dip below zero (same guard as
-  // src/efficiency-model.js).
+  // Clamp: busyCoreMs is measured from task events while the allocation comes from executor
+  // events, so the difference can dip below zero (same guard as efficiency-model.ts).
   const wastedCoreHours = Math.max(0, totalCoreHours - usefulCoreHours);
 
   const topStages = Object.entries(runAggregates.perStage ?? {})