sparkforensics-cli 0.1.0

package/vendor-core/stage-quantiles.js

@@ -0,0 +1,199 @@
+// Single source of truth for the packed per-task numeric array: FIELDS (offset
+// constants), TASK_FIELD_NAMES (display labels), and the finalizeStage hot-loop
+// push order are all derived from this list. `prop` is the task record's real
+// property name, kept separate from `name` since MEM_SPILLED's display label
+// ('memorySpilled') differs from it (memSpilled).
+                               
+              
+               
+               
+ 
+
+const TASK_FIELD_DESCRIPTORS                        = [
+  { key: 'DURATION', name: 'duration', prop: 'duration' },
+  { key: 'GC_TIME', name: 'gcTime', prop: 'gcTime' },
+  { key: 'MEM_SPILLED', name: 'memorySpilled', prop: 'memSpilled' },
+  { key: 'DISK_SPILLED', name: 'diskSpilled', prop: 'diskSpilled' },
+  { key: 'SHUFFLE_READ', name: 'shuffleRead', prop: 'shuffleRead' },
+  { key: 'SHUFFLE_WRITE', name: 'shuffleWrite', prop: 'shuffleWrite' },
+  { key: 'LAUNCH_TIME', name: 'launchTime', prop: 'launchTime' },
+  { key: 'FINISH_TIME', name: 'finishTime', prop: 'finishTime' },
+];
+
+const STRIDE = TASK_FIELD_DESCRIPTORS.length;
+
+export const FIELDS = Object.freeze({
+  ...Object.fromEntries(TASK_FIELD_DESCRIPTORS.map((d, i) => [d.key, i])),
+  STRIDE,
+})              
+                                                                            
+                                                              
+  ;
+
+export const TASK_FIELD_NAMES           = TASK_FIELD_DESCRIPTORS.map((d) => d.name);
+
+const TASK_FIELD_PROPS = TASK_FIELD_DESCRIPTORS.map((d) => d.prop);
+
+// Numeric accumulator fields on `stage` that this function reads/increments.
+// `stage`'s public parameter type stays the loose Record shape per this
+// module's Produces contract; this narrow view is only for the arithmetic
+// below, never a restatement of the object's real shape.
+                                   
+                           
+                            
+                        
+                             
+                           
+                    
+                          
+                          
+                     
+                      
+ 
+
+export function finalizeStage(
+  stageId        ,
+  stage                                                                                         ,
+  state                                          
+)                                                          {
+  // Already finalized (e.g. a duplicate/replayed StageCompleted event):
+  // degrade gracefully instead of crashing on stage.taskAttempts.values().
+  if (stage.taskAttempts === null) return null;
+
+  const acc = stage                                      ;
+
+  const buf           = [];
+  let taskCount = 0, failedTasks = 0, speculativeTasks = 0;
+  const hostStats = new Map();
+  const executorStats = new Map();
+  const failureReasons = new Map();
+  const localityStats = new Map();
+  let peakExecutionMemoryMax = 0;
+
+  for (const t of stage.taskAttempts.values()) {
+    taskCount++;
+    if (t.failed) {
+      failedTasks++;
+      if (t.reason) failureReasons.set(t.reason, (failureReasons.get(t.reason) ?? 0) + 1);
+    }
+    if (t.speculative) speculativeTasks++;
+    if (t.host) {
+      let hs = hostStats.get(t.host);
+      if (!hs) { hs = { taskCount: 0, totalDuration: 0 }; hostStats.set(t.host, hs); }
+      hs.taskCount++;
+      hs.totalDuration += t.duration;
+    }
+    if (t.executorId) {
+      let es = executorStats.get(t.executorId);
+      if (!es) { es = { taskCount: 0, totalDuration: 0, inputBytes: 0, shuffleReadBytes: 0, shuffleWriteBytes: 0 }; executorStats.set(t.executorId, es); }
+      es.taskCount++;
+      es.totalDuration += t.duration;
+      es.inputBytes += t.inputBytes;
+      es.shuffleReadBytes += t.shuffleRead;
+      es.shuffleWriteBytes += t.shuffleWrite;
+    }
+    if (t.locality) {
+      localityStats.set(t.locality, (localityStats.get(t.locality) ?? 0) + 1);
+    }
+    if (t.peakExecMem > peakExecutionMemoryMax) peakExecutionMemoryMax = t.peakExecMem;
+    acc.shuffleReadBytes += t.shuffleRead;
+    acc.shuffleWriteBytes += t.shuffleWrite;
+    acc.fetchWaitTime += t.fetchWaitTime;
+    acc.memoryBytesSpilled += t.memSpilled;
+    acc.diskBytesSpilled += t.diskSpilled;
+    acc.jvmGCTime += t.gcTime;
+    acc.executorRunTime += t.executorRunTime;
+    acc.executorCpuTime += t.executorCpuTime;
+    acc.inputBytes += t.inputBytes;
+    acc.outputBytes += t.outputBytes;
+    for (let i = 0; i < TASK_FIELD_PROPS.length; i++) buf.push(t[TASK_FIELD_PROPS[i]]);
+  }
+  stage.taskCount = taskCount;
+  stage.failedTasks = failedTasks;
+  stage.speculativeTasks = speculativeTasks;
+  stage.taskAttempts = null; // no longer needed after finalize, freeing memory
+
+  const arr = new Float64Array(buf);
+  state.taskStore.set(stageId, arr);
+
+  const { p50, p95, max } = computeDurationQuantiles(arr);
+  const spillClass = acc.memoryBytesSpilled > 0 ? classifySpill(arr) : 'unclassified';
+  const { p50: shuffleReadP50, p95: shuffleReadP95, max: shuffleReadMax } = computeFieldQuantiles(arr, FIELDS.SHUFFLE_READ);
+  const { p50: spillMemP50, p95: spillMemP95, max: spillMemMax } = computeFieldQuantiles(arr, FIELDS.MEM_SPILLED);
+  const { p50: spillDiskP50, p95: spillDiskP95, max: spillDiskMax } = computeFieldQuantiles(arr, FIELDS.DISK_SPILLED);
+
+  // Straggler count: tasks with duration > 4 * P50.
+  const stragglerThreshold = 4 * p50;
+  let stragglerCount = 0;
+  const taskArrCount = arr.length / FIELDS.STRIDE;
+  if (p50 > 0) {
+    for (let i = 0; i < taskArrCount; i++) {
+      if (arr[i * FIELDS.STRIDE + FIELDS.DURATION] > stragglerThreshold) stragglerCount++;
+    }
+  }
+
+  const hostStatsArr = [...hostStats.entries()].map(
+    ([host, s]) => ({ host, taskCount: s.taskCount, totalDuration: s.totalDuration })
+  );
+  const executorStatsArr = [...executorStats.entries()].map(
+    ([executorId, s]) => ({ executorId, taskCount: s.taskCount, totalDuration: s.totalDuration, inputBytes: s.inputBytes, shuffleReadBytes: s.shuffleReadBytes, shuffleWriteBytes: s.shuffleWriteBytes })
+  );
+  const failureReasonsArr = [...failureReasons.entries()].map(
+    ([reason, count]) => ({ reason, count })
+  );
+  const localityStatsArr = [...localityStats.entries()].map(
+    ([locality, count]) => ({ locality, count })
+  );
+
+  const data                          = {
+    ...stage,
+    hostStats: hostStatsArr, executorStats: executorStatsArr, failureReasons: failureReasonsArr, localityStats: localityStatsArr, stragglerCount,
+    peakExecutionMemoryMax,
+    taskDurationP50: p50,
+    taskDurationP95: p95,
+    taskDurationMax: max,
+    shuffleReadP50, shuffleReadP95, shuffleReadMax,
+    spillMemP50, spillMemP95, spillMemMax,
+    spillDiskP50, spillDiskP95, spillDiskMax,
+    gcPct: acc.executorRunTime > 0 ? (acc.jvmGCTime / acc.executorRunTime) * 100 : 0,
+    spillClassification: spillClass,
+    stageType: acc.shuffleReadBytes > 0 ? 'REDUCE' : 'MAP',
+  };
+  delete data.taskAttempts; // internal-only field, already nulled above; never part of the public message
+
+  return { type: 'stage', data };
+}
+
+export function computeFieldQuantiles(arr              , fieldIndex        )                                            {
+  const taskCount = arr.length / FIELDS.STRIDE;
+  if (taskCount === 0) return { p50: 0, p95: 0, max: 0 };
+
+  const values = new Float64Array(taskCount);
+  for (let i = 0; i < taskCount; i++) values[i] = arr[i * FIELDS.STRIDE + fieldIndex];
+  values.sort();
+
+  return {
+    p50: values[Math.ceil(taskCount * 0.50) - 1],
+    p95: values[Math.ceil(taskCount * 0.95) - 1],
+    max: values[taskCount - 1],
+  };
+}
+
+export function computeDurationQuantiles(arr              )                                            {
+  return computeFieldQuantiles(arr, FIELDS.DURATION);
+}
+
+export function classifySpill(arr              )                                     {
+  const taskCount = arr.length / FIELDS.STRIDE;
+  if (taskCount === 0) return 'unclassified';
+
+  let zeroCount = 0;
+  for (let i = 0; i < taskCount; i++) {
+    if (arr[i * FIELDS.STRIDE + FIELDS.MEM_SPILLED] === 0) zeroCount++;
+  }
+
+  const zeroFraction = zeroCount / taskCount;
+  if (zeroFraction >= 0.80) return 'skew';
+  if (zeroFraction < 0.20) return 'volume';
+  return 'unclassified';
+}

package/vendor-core/threshold-summary.js

@@ -0,0 +1,35 @@
+const THRESHOLD_SUMMARIES                         = {
+  spill: 'single-task disk spill above 1 GiB',
+  shuffle: 'shuffle read above the configured minimum byte threshold',
+  skew: 'task duration skew above the configured ratio',
+  gc: 'JVM GC time share above the configured ratio',
+  slowHost: 'a host running 2x+ slower than its peers by mean task duration (per-executor byte/time dimensions use a separate, narrower ratio ladder starting at 1.33x; only those can reach critical on ratio alone)',
+  stageSlowness: 'a stage running far longer than its peers, not attributable to a single slow host',
+  straggler: 'one or more tasks finishing far after the rest of their stage',
+  speculationWaste: 'speculative task attempts that completed after the original',
+  tinyTask: 'median task duration below the configured floor',
+  partitionSizing: 'partition byte size outside the configured target range',
+  stageShape: 'low parallelism, data explosion, or task-count skew relative to core count',
+  stageFailed: 'a stage that failed outright',
+  failures: 'task failures above the configured rate',
+  retryWaste: 'retried task attempts consuming executor time',
+  coldStart: 'executor startup time above the configured floor',
+  incompleteRun: 'an event log missing its terminal ApplicationEnd/job-completion event',
+  utilization: 'core occupancy below the configured floor across the run',
+  memoryUtilization: 'executor heap usage outside the configured band',
+  cacheUtilization: 'cached partitions evicted or spilled to disk',
+  coreLocality: 'task placement missing data-local core assignment',
+  cachingOpportunity: 'a dataset re-read from source multiple times with no cache/persist',
+  jobFailureRate: 'job failure rate above the configured threshold',
+  autoscalingChurn: 'executor add/remove churn above the configured rate',
+  configAudit: 'a Spark conf value outside the recommended range',
+  duplicatePlanSubtree: 'the same physical plan subtree executed more than once',
+  smallFiles: 'output files below the configured target size',
+  overBroadcast: 'a broadcast join above the configured size ceiling',
+  underBroadcast: 'a join below the configured size floor that skipped broadcast',
+  broadcastSizing: 'a broadcast join outside the configured size range in either direction',
+};
+
+export function getThresholdSummary(type        )         {
+  return THRESHOLD_SUMMARIES[type] ?? 'criteria not met';
+}

package/vendor-core/types.js

@@ -0,0 +1,286 @@
+                             
+                                                         
+
+                               
+                
+              
+                                                                        
+                                                                        
+                                         
+                               
+                                  
+                                      
+                                  
+                     
+                                                                   
+                                                                   
+                          
+                         
+ 
+
+                                 
+                             
+                          
+                           
+                              
+                        
+                           
+                             
+                      
+ 
+
+                         
+                                                                                
+                                                                               
+                                                                                                                    
+                                
+                                                                                                 
+                          
+                                                                                    
+                                                                             
+// Contract-pinned evidence `eventType`: the same fixed set as the parser's
+// `evidenceInputs` counter keys, so provenance stays addressable against the
+// documented `{ eventType, count }` shape (review: values must not diverge
+// from the counters they summarize).
+                               
+                                                                                         
+                                                                                
+                                            
+                   
+                       
+                                 
+                  
+                                                             
+ 
+                                                                                                 
+
+                        
+              
+                
+                     
+                                 
+                             
+                       
+                       
+                       
+                     
+                       
+                      
+                       
+                            
+                          
+                          
+                             
+                         
+                              
+                            
+                                                           
+                     
+                           
+                 
+                           
+                           
+                           
+                                  
+                                                        
+                   
+                         
+ 
+
+                           
+               
+                  
+                                                                   
+                       
+                      
+ 
+
+                               
+                      
+                             
+                                   
+                         
+ 
+                                    
+             
+                       
+                
+                   
+                         
+                        
+                       
+                                     
+                                                                    
+                                                                         
+                               
+ 
+
+                                
+             
+                 
+                 
+ 
+
+                                 
+                             
+                         
+                                                                         
+                                                                             
+                                                                   
+                              
+                       
+                                                                           
+                                                                     
+                                                                        
+                            
+                                                                            
+                                                                          
+                                                                             
+                                                              
+                                       
+ 
+
+                                                              
+
+                      
+             
+                                
+                     
+                                
+                        
+                            
+                     
+                                
+ 
+
+                                     
+                
+                    
+                     
+               
+                     
+                                   
+ 
+                                       
+                  
+                    
+                     
+                 
+ 
+                                                                      
+                                
+                          
+                      
+                               
+                                                                                 
+                         
+ 
+
+                           
+                           
+                              
+                                                                              
+                                 
+                         
+                                      
+                                                    
+ 
+
+                                                                   
+
+                                                                                 
+
+                                 
+                
+                     
+ 
+
+                                                                                            
+
+/** Static, hand-authored tag for how much work a finding's fix requires:
+ * a conf/spark-submit flag change, a Spark job edit, or a pipeline restructure. */
+                                                          
+
+                                 
+                             
+                                                                               
+                                                                         
+                         
+                                                  
+                                       
+                            
+ 
+
+                          
+              
+                           
+               
+                                                                        
+                                                                          
+                                                                          
+                                                                      
+                                                                            
+                                                                          
+                                                                          
+                                                                           
+                                                                        
+                                                                          
+                                
+                           
+                         
+                  
+                                                                        
+                                                                      
+                                                                            
+                                                                        
+                                                                       
+                                                                      
+                                                                   
+                                                                    
+                                                       
+                          
+                          
+                     
+                      
+                              
+                                  
+                    
+                                          
+                    
+                  
+                          
+                          
+                                                     
+                              
+                                                                                    
+                      
+                         
+ 
+
+                                                                                     
+
+                                                
+                
+                 
+                         
+                                                                             
+                                                                         
+                                                                           
+                   
+                               
+                                                                               
+                      
+ 
+
+                                              
+                
+                 
+                  
+                      
+                                                                          
+                                                                           
+                                                                       
+                                       
+                            
+                        
+