sparkforensics-mcp 0.3.0 → 0.5.0

package/vendor-core/threshold-overrides.js

@@ -0,0 +1,160 @@
+// User threshold overrides: validation of a parsed config file, and which of a detector's
+// thresholds an override actually moved off its specification default. Only the CLI and the MCP
+// server accept overrides; the dashboard always runs the defaults. Reading the file is
+// cli/threshold-config.ts's job, so this module stays free of Node APIs.
+import {
+  DETECTORS, ENTRY_BY_TYPE, detectorCatalog,                                                                                            
+} from './detectors.js';
+                                                                  
+
+const entries                      = DETECTORS;
+
+function isPlainObject(value         )                                   {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isNonNegativeNumber(value         )                  {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0;
+}
+
+// Entries a user may tune: not config-scope (those audits compare against Spark's own defaults,
+// such as its memoryOverhead floor), and with at least one threshold.
+function tunableEntry(type        )           {
+  const entry = entries.find((d) => d.type === type);
+  if (!entry) {
+    const tunable = entries.filter((d) => d.scope !== 'config' && Object.keys(d.thresholds).length > 0).map((d) => d.type);
+    throw new Error(`unknown detector "${type}" (tunable detectors: ${tunable.join(', ')})`);
+  }
+  if (entry.scope === 'config') throw new Error(`"${type}" is not tunable: its checks compare against Spark's own defaults`);
+  if (Object.keys(entry.thresholds).length === 0) throw new Error(`"${type}" has no thresholds to tune`);
+  return entry;
+}
+
+function checkValue(type        , name        , fallback                            , value         )                             {
+  if (!Array.isArray(fallback)) {
+    if (!isNonNegativeNumber(value)) throw new Error(`"${type}.${name}" must be a non-negative number`);
+    return value;
+  }
+  // Tier tables are indexed by position and read as ascending bands.
+  const ascending = Array.isArray(value) && value.length === fallback.length
+    && value.every(isNonNegativeNumber) && value.every((v, i) => i === 0 || v >= value[i - 1]);
+  if (!ascending) throw new Error(`"${type}.${name}" must be an ascending list of ${fallback.length} non-negative numbers`);
+  return Object.freeze([...value]);
+}
+
+/**
+ * Validates a parsed thresholds config: `{ "<detector type>": { "<threshold>": value } }`, with each
+ * value shaped like that threshold's default (see a report's `detectors` catalog for the names,
+ * units and defaults). Throws an Error naming the first problem; never drops a bad entry silently.
+ */
+export function parseThresholdOverrides(raw         )                     {
+  if (!isPlainObject(raw)) throw new Error('expected a JSON object keyed by detector type');
+  const result                                     = {};
+  for (const [type, perDetector] of Object.entries(raw)) {
+    const entry = tunableEntry(type);
+    if (!isPlainObject(perDetector)) throw new Error(`"${type}" must be an object of threshold values`);
+    const values                                             = {};
+    for (const [name, value] of Object.entries(perDetector)) {
+      // Own keys only: an inherited name such as `constructor` or `toString` is no threshold.
+      if (!Object.hasOwn(entry.thresholds, name)) {
+        throw new Error(`unknown threshold "${type}.${name}" (${type} thresholds: ${Object.keys(entry.thresholds).join(', ')})`);
+      }
+      values[name] = checkValue(type, name, entry.thresholds[name], value);
+    }
+    result[type] = Object.freeze(values);
+  }
+  return Object.freeze(result)                      ;
+}
+
+/** The overrides for one entry, as the untyped map withThresholds() takes. */
+export function overridesFor(entry          , overrides                                )                                 {
+  return (overrides                                                            )?.[entry.type];
+}
+
+function sameValue(a                            , b                            )          {
+  if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((v, i) => v === b[i]);
+  return a === b;
+}
+
+/** The entry's thresholds that `overrides` moves off the default, or null when none does (no
+ * override, or one equal to the default: that run is the specification's). */
+export function tunedThresholdsOf(entry          , overrides                                )                         {
+  const tuned                                 = {};
+  for (const [name, value] of Object.entries(overridesFor(entry, overrides) ?? {})) {
+    if (!Object.hasOwn(entry.thresholds, name)) continue;
+    const fallback = entry.thresholds[name];
+    if (!sameValue(value, fallback)) tuned[name] = { value, default: fallback };
+  }
+  return Object.keys(tuned).length > 0 ? tuned : null;
+}
+
+/** The tuned thresholds an entry's findings carry: its own, plus its `suppressedBy` entry's
+ * (named `<suppressor>.<threshold>`), since tuning the suppressor changes which of them survive. */
+export function findingTunedThresholds(entry          , overrides                                )                         {
+  const own = tunedThresholdsOf(entry, overrides);
+  const suppressor = entry.suppressedBy ? entries.find((d) => d.type === entry.suppressedBy) : undefined;
+  const bySuppressor = suppressor ? tunedThresholdsOf(suppressor, overrides) : null;
+  if (!suppressor || !bySuppressor) return own;
+  const prefixed = Object.fromEntries(Object.entries(bySuppressor).map(([name, t]) => [`${suppressor.type}.${name}`, t]));
+  return { ...own, ...prefixed };
+}
+
+/** findingTunedThresholds() for the first entry emitting finding type `type`, the entry whose
+ * thresholds its clean-check summary reads. */
+export function tunedThresholdsForType(type        , overrides                                )                         {
+  const entry = ENTRY_BY_TYPE.get(type);
+  return entry ? findingTunedThresholds(entry, overrides) : null;
+}
+
+/** Every tuned detector's overridden thresholds, keyed by entry type, or null when none is tuned. */
+export function tunedDetectors(overrides                                )                                         {
+  const byType                                  = {};
+  for (const entry of entries) {
+    const tuned = tunedThresholdsOf(entry, overrides);
+    if (tuned) byType[entry.type] = tuned;
+  }
+  return Object.keys(byType).length > 0 ? byType : null;
+}
+
+/** The thresholds an entry runs with under `overrides`: its own, with any override merged in. */
+export function effectiveThresholds(entry          , overrides                                )                     {
+  const own = overridesFor(entry, overrides);
+  return own ? { ...entry.thresholds, ...own } : entry.thresholds;
+}
+
+/** detectorCatalog() as a run under `overrides` used it: each row's effective thresholds, plus
+ * `tunedThresholds` on a row an override moved off its defaults. */
+export function tunedDetectorCatalog(overrides                                )                         {
+  return detectorCatalog().map((row, i) => {
+    const entry = entries[i];
+    const tuned = tunedThresholdsOf(entry, overrides);
+    return tuned ? { ...row, thresholds: effectiveThresholds(entry, overrides), tunedThresholds: tuned } : row;
+  });
+}
+
+function formatThresholdValue(value                            )         {
+  return Array.isArray(value) ? `[${value.join(', ')}]` : String(value);
+}
+
+/** "ratioWarn 5 (default 3), minTasksForP95 40 (default 20)". */
+export function describeTunedThresholds(tuned                 )         {
+  return Object.entries(tuned)
+    .map(([name, { value, default: fallback }]) => `${name} ${formatThresholdValue(value)} (default ${formatThresholdValue(fallback)})`)
+    .join(', ');
+}
+
+/** A tuned run's report line after its label: every tuned detector's thresholds, then why its
+ * findings' estimates are uncalibrated. `byType` is tunedDetectors()'s result. */
+export function tunedRunNote(byType                                 )         {
+  const tuned = Object.entries(byType).map(([type, t]) => `${type} ${describeTunedThresholds(t)}`).join('; ');
+  return `${tuned}. Findings from these detectors are marked, and their impact estimates are uncalibrated: the estimates are calibrated against the default thresholds.`;
+}
+
+/** The caveat a tuned finding carries in `validationRequired`: which thresholds produced it and,
+ * when it has an estimate figure (`hasEstimate`), that the figure was never calibrated. */
+export function tunedThresholdsNote(tuned                 , hasEstimate         )         {
+  const label = `Produced with tuned thresholds: ${describeTunedThresholds(tuned)}.`;
+  return hasEstimate
+    ? `${label} Impact estimates are calibrated against the default thresholds, so this finding's estimate is unvalidated.`
+    : label;
+}

package/vendor-core/threshold-summary.js

@@ -1,35 +1,13 @@
-const THRESHOLD_SUMMARIES                         = {
-  spill: 'single-task disk spill above 1 GiB',
-  shuffle: 'shuffle read above the configured minimum byte threshold',
-  skew: 'task duration skew above the configured ratio',
-  gc: 'JVM GC time share above the configured ratio',
-  slowHost: 'a host running 2x+ slower than its peers by mean task duration (per-executor byte/time dimensions use a separate, narrower ratio ladder starting at 1.33x; only those can reach critical on ratio alone)',
-  stageSlowness: 'a stage running far longer than its peers, not attributable to a single slow host',
-  straggler: 'one or more tasks finishing far after the rest of their stage',
-  speculationWaste: 'speculative task attempts that completed after the original',
-  tinyTask: 'median task duration below the configured floor',
-  partitionSizing: 'partition byte size outside the configured target range',
-  stageShape: 'low parallelism, data explosion, or task-count skew relative to core count',
-  stageFailed: 'a stage that failed outright',
-  failures: 'task failures above the configured rate',
-  retryWaste: 'retried task attempts consuming executor time',
-  coldStart: 'executor startup time above the configured floor',
-  incompleteRun: 'an event log missing its terminal ApplicationEnd/job-completion event',
-  utilization: 'core occupancy below the configured floor across the run',
-  memoryUtilization: 'executor heap usage outside the configured band',
-  cacheUtilization: 'cached partitions evicted or spilled to disk',
-  coreLocality: 'task placement missing data-local core assignment',
-  cachingOpportunity: 'a dataset re-read from source multiple times with no cache/persist',
-  jobFailureRate: 'job failure rate above the configured threshold',
-  autoscalingChurn: 'executor add/remove churn above the configured rate',
-  configAudit: 'a Spark conf value outside the recommended range',
-  duplicatePlanSubtree: 'the same physical plan subtree executed more than once',
-  smallFiles: 'output files below the configured target size',
-  overBroadcast: 'a broadcast join above the configured size ceiling',
-  underBroadcast: 'a join below the configured size floor that skipped broadcast',
-  broadcastSizing: 'a broadcast join outside the configured size range in either direction',
-};
+import { ENTRY_BY_TYPE,                         } from './detectors.js';
+import { presentationOf } from './finding-presentation.js';
+import { effectiveThresholds } from './threshold-overrides.js';
 
-export function getThresholdSummary(type        )         {
-  return THRESHOLD_SUMMARIES[type] ?? 'criteria not met';
+/** The clean-check criterion for a finding type, built from the thresholds of the first DETECTORS
+ * entry that emits it (configAudit's four entries share one summary), with `overrides` merged in
+ * when the run was tuned. */
+export function getThresholdSummary(type        , overrides                     )         {
+  const presentation = presentationOf(type);
+  const entry = ENTRY_BY_TYPE.get(type);
+  if (!presentation || !entry) return 'criteria not met';
+  return presentation.thresholdSummary(effectiveThresholds(entry, overrides)         );
 }

package/vendor-core/types.js

@@ -1,3 +1,6 @@
+                                                  
+                                                                
+
                              
                                                          
 
@@ -54,6 +57,24 @@
  
                                                                                                  
 
+                                     
+                    
+                      
+                         
+                          
+                          
+                    
+                             
+                           
+                           
+                            
+                     
+                      
+                               
+                                 
+                            
+ 
+
                         
               
                 
@@ -67,6 +88,20 @@
                        
                       
                        
+                                                                                                  
+                                
+                                                                                        
+                                                                    
+                         
+                               
+                                                                                               
+                                                                                       
+                                              
+                                                                                                
+                                                                                            
+                                                                                                  
+                             
+                                              
                             
                           
                           
@@ -260,6 +295,10 @@
                          
                                       
                                                     
+                                                                               
+                        
+                                                                                
+                                     
  
 
                                                                    
@@ -269,6 +308,9 @@
                                  
                 
                      
+                                                                                         
+                                                                                    
+              
  
 
                                                                                             
@@ -285,50 +327,20 @@
                                                   
                                        
                             
+                                                                                               
+                                                                                               
+                                                                                             
+                                                                                                 
+                                                                                               
+                                                                                   
+                                                                                                                                  
+                                                                     
+                                                    
  
 
-                          
-              
-                           
-               
-                                                                            
-                                                                  
-                                                                            
-                                                                              
-                                                                             
-                                                       
-                           
-                         
-                  
-                                                                        
-                                                                          
-                                                                            
-                                                                         
-                                                                              
-                    
-                          
-                          
-                     
-                      
-                              
-                                  
-                    
-                                          
-                    
-                  
-                          
-                          
-                                                     
-                              
-                                                                                    
-                      
-                                                                        
-                                                                             
-                                                                            
-                                                                         
-                         
-                         
- 
+// `Finding` is a union discriminated on `type`, one member per emitted finding type: see
+// finding-types.ts for each detector's shape and which of its fields are public evidence.
+                                                                                                                               
 
                                                                                      
 

package/vendor-core/wall-clock.js

@@ -1,15 +1,4 @@
-export function mergeIntervals(intervals                    )                     {
-  if (intervals.length === 0) return [];
-  const sorted = [...intervals].sort((a, b) => a[0] - b[0]);
-  const out                     = [[sorted[0][0], sorted[0][1]]];
-  for (let i = 1; i < sorted.length; i++) {
-    const last = out[out.length - 1];
-    const cur = sorted[i];
-    if (cur[0] <= last[1]) last[1] = Math.max(last[1], cur[1]);
-    else out.push([cur[0], cur[1]]);
-  }
-  return out;
-}
+import { mergeIntervals } from './intervals.js';
 
 export function computeWallClock(app                                                             , stages                                                           )                                                                                     {
   const start = app?.startTime ?? 0;

package/vendor-core/wasted-core-hours.js

@@ -2,12 +2,12 @@
 // the memoryUtilization detector's idle-cores math uses: capacity core-time vs.
 // core-time that actually ran tasks. Feeds a report widget only, no detector.
 
-import { computeTotalCores } from './core-count.js';
+import { computePeakConcurrentCores } from './core-count.js';
+import { MS_PER_CORE_HOUR } from './format-utils.js';
 
-export const MS_PER_CORE_HOUR         = 3.6e6;
 const TOP_N = 5;
 
-                                 
+                                        
                                 
                                  
                                  
@@ -30,21 +30,24 @@ const EMPTY                        = {
 
 // `app`/`runAggregates` are nullable because real callers pass null (app is
 // SparkAppInfo | null before parsing completes), which the guard below already
-// tolerates. `resources` is on app's shape so the computeTotalCores pass-through
-// type-checks; real app objects always carry it.
+// tolerates. `resources` is on app's shape so the computePeakConcurrentCores
+// pass-through type-checks; real app objects always carry it. An omitted
+// `executorsRemoved` means no executor left, where peak cores equal the sum.
 export function computeWastedCoreHours(
   app                                                                                                       ,
-  executorsAdded                                             = [],
+  executorsAdded                                                                                    = [],
   runAggregates                      ,
+  executorsRemoved                                                   = [],
 )                        {
   // Nullish (not falsy) guard on times: a literal startTime:0 is valid.
   if (!runAggregates || app?.startTime == null || app?.endTime == null) return EMPTY;
   const appDurationMs = app.endTime - app.startTime;
   if (appDurationMs <= 0) return EMPTY;
 
-  // Total cores: prefer real Executor-Added Total Cores, else peakExecutors ×
-  // configured cores (same fallback as the memoryUtilization detector).
-  const totalCores = computeTotalCores(app, executorsAdded);
+  // Total cores: peak concurrent Executor-Added Total Cores, else peak executors ×
+  // configured cores. Same capacity as the memoryUtilization detector, so an
+  // executor replaced mid-run is not counted twice.
+  const totalCores = computePeakConcurrentCores(app, executorsAdded, executorsRemoved);
   if (totalCores <= 0) return EMPTY;
 
   const totalCoreHours = (totalCores * appDurationMs) / MS_PER_CORE_HOUR;