sparkforensics-mcp 0.3.0 → 0.5.0

package/vendor-core/finding-presentation.js

@@ -0,0 +1,368 @@
+// How each finding type is presented: the one place a finding type's name, board tag, action
+// label, clean-check threshold summary and generic recommendation are registered. FINDING_NAMES,
+// TYPE_TAG_MAP, getThresholdSummary, findingActionLabel and coreFindingGenericRecommendation all
+// read this table.
+//
+// It sits beside DETECTORS rather than on its entries because the HTML export renders names, tags
+// and labels but may not reach detectors.ts (scripts/export-analysis-guard.mjs). The import from
+// detectors.ts is type-only, so it is erased at build time. A detector's scope, order and emits
+// list stay on its DETECTORS entry; renderers get them through detectorInfoByType().
+                                                   
+                                                      
+                                                         
+
+                                                             
+                                                                                          
+               
+                                                                                             
+              
+                                                                                                    
+                                                        
+                                                                                             
+                                                                                              
+                                                         
+                                                                                                 
+                                                                                                   
+                                                                                        
+                                                                   
+ 
+
+/** A share threshold as captions and caveats state it: 0.005 -> "0.5%", never float noise like 7.000000000000001%. */
+export const shareLabel = (share        )         => `${Math.round(share * 1e6) / 1e4}%`;
+
+// Whether the detector found `key` already logged on for this run. Its switchFix (detectors.ts)
+// then worded the row's own text for that case and left the property out of the remediation, so
+// a generic line reads the same decision and never recommends a switch the row says is on.
+// A finding with no remediation (older or hand-built data) keeps the property wording.
+function switchAlreadyOn(finding                                 , key        )          {
+  return finding.remediation != null && !finding.remediation.some((r) => r.key === key);
+}
+
+const SKEW_JOIN_KEY = 'spark.sql.adaptive.skewJoin.enabled';
+const SKEW_JOIN_ALREADY_ON = 'AQE skew-join handling is already on, so salt the key or repartition on a better key.';
+const SKEW_JOIN_AQE_OFF = 'AQE is off, so enable it (spark.sql.adaptive.enabled) for skew-join handling to apply; otherwise salt the key or repartition on a better key.';
+
+// The skew-join generic line, worded per the row's remediation: AQE logged off, switch already on, or neither.
+function skewJoinGeneric(f                                 , unset        )         {
+  if (f.remediation?.some((r) => r.key === 'spark.sql.adaptive.enabled')) return SKEW_JOIN_AQE_OFF;
+  return switchAlreadyOn(f, SKEW_JOIN_KEY) ? SKEW_JOIN_ALREADY_ON : unset;
+}
+const DYNAMIC_ALLOCATION_KEY = 'spark.dynamicAllocation.enabled';
+
+// The four configAudit DETECTORS entries share this row, one per audited property.
+const CONFIG_AUDIT_PRESENTATION                                     = {
+  name: 'config audit',
+  tag: 'CFG',
+  thresholdSummary: () => 'a Spark conf value outside the recommended range',
+  actionLabel(f) {
+    switch (f.property) {
+      case 'spark.shuffle.service.enabled': return 'Enable shuffle service';
+      case 'spark.dynamicAllocation.minExecutors': return 'Fix autoscaling bounds';
+      case 'spark.dynamicAllocation.maxExecutors': return 'Set max executors';
+      case 'spark.serializer': return 'Switch to Kryo';
+      case 'spark.executor.memoryOverhead': return 'Raise memory overhead';
+    }
+    return undefined;
+  },
+  genericRecommendation(f) {
+    switch (f.property) {
+      case 'spark.shuffle.service.enabled': return 'Set spark.shuffle.service.enabled=true so shuffle data survives executor removal.';
+      case 'spark.dynamicAllocation.minExecutors': return 'Set the minimum executor bound at or below the maximum.';
+      case 'spark.dynamicAllocation.maxExecutors': return 'Set spark.dynamicAllocation.maxExecutors to cap cluster growth.';
+      case 'spark.serializer': return 'Consider spark.serializer=org.apache.spark.serializer.KryoSerializer for faster, smaller buffers.';
+      case 'spark.executor.memoryOverhead': return 'Raise executor memoryOverhead above Spark\'s default floor to avoid off-heap OOM-kills.';
+    }
+    return undefined;
+  },
+};
+
+/** One row per emitted finding type: the mapped type makes a missing or stray row a compile error. */
+export const FINDING_PRESENTATION                                                          = {
+  incompleteRun: {
+    name: 'incomplete run',
+    tag: 'INCMP',
+    thresholdSummary: () => 'an event log missing its terminal ApplicationEnd event',
+    actionLabel: () => undefined,
+    genericRecommendation: () => undefined,
+  },
+
+  skew: {
+    name: 'task skew',
+    tag: 'SKEW',
+    thresholdSummary: (t) => `P95 task time over ${t.ratioWarn}× the median (the longest task on stages under ${t.minTasksForP95} tasks)`,
+    actionLabel: () => 'Fix task skew',
+    genericRecommendation: (f) => skewJoinGeneric(f,
+      'For join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.'),
+  },
+  stageShape: {
+    name: 'stage shape',
+    tag: 'SHAPE',
+    thresholdSummary: (t) => `under ${t.pRatioMax} tasks per core, output over ${t.oiRatioMax}× input, or one task spanning over ${Math.round(t.stageShareMin * 100)}% of the stage's wall-clock at over ${t.skewWarn}× the median task`,
+    actionLabel(f) {
+      switch (f.rule) {
+        case 'lowParallelism': return 'Increase parallelism';
+        case 'dataExplosion': return 'Check for exploding join';
+        case 'taskStageSkew': return 'Fix straggler task';
+      }
+      return undefined;
+    },
+    genericRecommendation(f) {
+      switch (f.rule) {
+        case 'lowParallelism': return 'Too few tasks run relative to the cores available, leaving cluster capacity idle: repartition to use more of it.';
+        case 'dataExplosion': return 'Output volume far exceeds input volume: check for an exploding join or a cross product.';
+        case 'taskStageSkew': return 'A single straggler task gates the whole stage\'s wall-clock duration.';
+      }
+      return undefined;
+    },
+  },
+  tinyTask: {
+    name: 'tiny tasks',
+    tag: 'TINY',
+    thresholdSummary: (t) => `${t.minTasks}+ tasks with a median of ${t.maxP50}ms or less and a P95 of ${t.maxP95}ms or less`,
+    actionLabel: () => 'Coalesce small tasks',
+    // The shuffle-vs-no-shuffle fix isn't a Finding field, so one sentence covers both.
+    genericRecommendation: () => 'Scheduler overhead may dominate: lower spark.sql.shuffle.partitions, or coalesce down to fewer, larger tasks.',
+  },
+
+  shuffle: {
+    name: 'shuffle I/O',
+    tag: 'SHFL',
+    thresholdSummary: (t) => `stage shuffle read above ${t.minBytes / 1048576} MiB`,
+    actionLabel: () => 'Reduce shuffle size',
+    genericRecommendation: () => 'Raise spark.sql.shuffle.partitions, or use a broadcast join for the smaller side.',
+  },
+  partitionSizing: {
+    name: 'partition sizing',
+    tag: 'PART',
+    thresholdSummary: (t) => `a shuffle partition over ${t.skewRatio}× the median or over ${t.maxPartBytes / 1073741824} GiB, or ${t.lowParTotalBytes / 1073741824} GiB of shuffle on ${t.lowParMaxTasks} tasks or fewer`,
+    actionLabel(f) {
+      switch (f.rule) {
+        case 'shufflePartitionSkew': return 'Fix skewed partition';
+        case 'lowShuffleParallelism': return 'Add shuffle partitions';
+        case 'maxPartitionTooBig': return 'Repartition oversized data';
+      }
+      return undefined;
+    },
+    genericRecommendation(f) {
+      switch (f.rule) {
+        case 'shufflePartitionSkew': return skewJoinGeneric(f, 'For join skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.');
+        case 'lowShuffleParallelism': return switchAlreadyOn(f, 'spark.sql.shuffle.partitions')
+          ? "spark.sql.shuffle.partitions is already high enough, so raise this stage's own partition count (its repartition(n) or RDD parallelism) so each partition is smaller."
+          : 'Raise spark.sql.shuffle.partitions so each partition is smaller.';
+        case 'maxPartitionTooBig': return 'Repartition to break up the oversized partition before this stage.';
+      }
+      return undefined;
+    },
+  },
+
+  spill: {
+    name: 'spill',
+    tag: 'SPILL',
+    thresholdSummary: (t) => `single-task disk spill above ${t.singleTaskDiskGiB} GiB`,
+    actionLabel: () => 'Reduce spill',
+    // The skew/volume classification isn't a Finding field, so one sentence covers both.
+    genericRecommendation: () => 'If the spill is skew-driven, fix task skew first: adding memory will not help. Otherwise raise spark.sql.shuffle.partitions or increase executor memory.',
+  },
+
+  gc: {
+    name: 'GC pressure',
+    tag: 'GC',
+    thresholdSummary: (t) => `GC above ${t.warnPct100}% (or below ${t.lowInfoPct100}%) of executor run time`,
+    actionLabel: (f) => (f.direction === 'low' ? 'Right-size executor memory' : 'Reduce GC pressure'),
+    genericRecommendation: (f) => (f.direction === 'low'
+      ? 'Memory may be over-provisioned here: consider reducing spark.executor.memory for cost savings.'
+      : 'Reduce object creation, use primitive types, avoid UDFs, or increase executor memory to cut GC time.'),
+  },
+
+  stageFailed: {
+    name: 'failed stage',
+    tag: 'SFAIL',
+    thresholdSummary: () => 'a stage that failed outright',
+    actionLabel: () => 'Inspect stage failure',
+    genericRecommendation: () => 'Inspect the driver log for the failure reason and the job that triggered it.',
+  },
+  failures: {
+    name: 'failed tasks',
+    tag: 'FAIL',
+    thresholdSummary: (t) => `over ${shareLabel(t.warnRate)} of a stage's tasks failing`,
+    actionLabel: () => 'Investigate task failures',
+    genericRecommendation: () => 'Investigate driver logs for executor instability or data-driven errors.',
+  },
+  retryWaste: {
+    name: 'retry waste',
+    tag: 'RETRY',
+    thresholdSummary: (t) => `${t.minWasted}+ retried attempts wasting at least ${t.minWasteMs / 1000}s`,
+    actionLabel: () => 'Investigate retry cause',
+    genericRecommendation: () => 'Investigate executor loss or fetch failures behind the retried attempts.',
+  },
+
+  slowHost: {
+    name: 'slow executor host',
+    tag: 'HOST',
+    thresholdSummary: (t) => `a host ${t.ratioWarn}× slower than its peers by mean task time (per-executor figures from ${t.ratioTiers[0]}×)`,
+    actionLabel(f) {
+      if (f.variant === 'durationShare') return 'Fix data locality';
+      if (f.variant === 'multiDim') return 'Investigate degraded executor';
+      return 'Check slow host';
+    },
+    genericRecommendation(f) {
+      if (f.variant === 'durationShare') return 'Check for data locality or partition assignment skewing work onto one node.';
+      if (f.variant === 'multiDim') return 'Investigate uneven partition assignment or a degraded executor.';
+      const check = 'Check what this host was running: it may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault.';
+      return switchAlreadyOn(f, 'spark.speculation')
+        ? `${check} Speculation is already on, so a lagging task there is already relaunched.`
+        : `${check} Enable spark.speculation to relaunch a lagging task automatically.`;
+    },
+  },
+  stageSlowness: {
+    name: 'slow stage',
+    tag: 'SLOW',
+    thresholdSummary: () => 'a stage running far longer than its peers, not attributable to a single slow host',
+    actionLabel: () => 'Profile slow stage',
+    genericRecommendation: () => 'Often a partition-count problem: raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.',
+  },
+  straggler: {
+    name: 'straggling task',
+    tag: 'STRAG',
+    thresholdSummary: () => 'one or more tasks finishing far after the rest of their stage',
+    actionLabel: () => 'Fix stragglers',
+    genericRecommendation: () => 'Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue. If a skewed key is the real cause, that is a candidate for AQE\'s skew-join handling.',
+  },
+  speculationWaste: {
+    name: 'speculation waste',
+    tag: 'SPEC',
+    thresholdSummary: (t) => `${t.minWasted}+ discarded speculative attempts wasting at least ${t.minWasteMs / 1000}s`,
+    actionLabel: () => 'Tune speculation settings',
+    genericRecommendation: () => 'If task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.',
+  },
+  coldStart: {
+    name: 'cold start',
+    tag: 'COLD',
+    thresholdSummary: (t) => `the first stage waiting over ${t.gapSeconds}s for an executor`,
+    actionLabel: () => 'Pre-warm cluster',
+    genericRecommendation: () => 'Keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it does not scale up from zero.',
+  },
+
+  memoryUtilization: {
+    name: 'memory utilization',
+    tag: 'MEM',
+    thresholdSummary: () => 'executor heap usage outside the configured band',
+    actionLabel(f) {
+      switch (f.variant) {
+        case 'idleCores': return 'Reduce idle cores';
+        case 'wasteModel': return 'Right-size executor memory';
+        case 'memoryBand':
+          if (f.dataUnavailable) return 'Enable memory metrics';
+          return f.rule === 'heapNearCapacity' ? 'Increase executor memory' : 'Reduce executor memory';
+      }
+      return undefined;
+    },
+    genericRecommendation(f) {
+      switch (f.variant) {
+        case 'idleCores': return switchAlreadyOn(f, DYNAMIC_ALLOCATION_KEY)
+          ? 'Dynamic allocation is already on, so reduce cluster size.'
+          : 'Reduce cluster size or enable dynamic allocation.';
+        case 'wasteModel': return 'Review spark.executor.memory and executor count.';
+        case 'memoryBand':
+          if (f.dataUnavailable) return undefined;
+          return f.rule === 'heapNearCapacity'
+            ? 'Memory may be too small: raise spark.executor.memory to avoid OOM/spill.'
+            : 'Memory may be over-provisioned: consider reducing spark.executor.memory for cost savings.';
+      }
+      return undefined;
+    },
+  },
+  utilization: {
+    name: 'executor utilization',
+    tag: 'UTIL',
+    thresholdSummary: (t) => `average executor utilization below ${shareLabel(t.minUtil)}`,
+    actionLabel: () => 'Reduce cluster size',
+    genericRecommendation: (f) => (switchAlreadyOn(f, DYNAMIC_ALLOCATION_KEY)
+      ? 'Dynamic allocation is already on, so consider reducing cluster size.'
+      : 'Consider reducing cluster size or enabling dynamic allocation.'),
+  },
+  coreLocality: {
+    name: 'core locality',
+    tag: 'LOCAL',
+    thresholdSummary: () => 'task placement missing data-local core assignment',
+    actionLabel: () => 'Fix data locality',
+    genericRecommendation: () => 'Check executor/data colocation.',
+  },
+  cachingOpportunity: {
+    name: 'caching opportunity',
+    tag: 'CACHE',
+    thresholdSummary: () => 'a dataset re-read from source multiple times with no cache/persist',
+    actionLabel: (f) => (f.variant === 'composite' ? 'Cache repeated result' : 'Cache shared table'),
+    genericRecommendation: (f) => (f.variant === 'composite'
+      ? 'Cache or persist the repeated join/union result so it is computed once instead of recomputed per query.'
+      : 'Cache the shared DataFrame, or broadcast it if it is a small join lookup.'),
+  },
+  cacheUtilization: {
+    name: 'cache utilization',
+    tag: 'CSTOR',
+    thresholdSummary: () => 'cached partitions evicted or spilled to disk',
+    actionLabel: (f) => (f.dataUnavailable ? 'Enable block-update logging' : 'Increase cache memory'),
+    genericRecommendation(f) {
+      switch (f.variant) {
+        case 'partialCache': return 'Increase executor memory or reduce the cached dataset size so more of it stays cached.';
+        case 'diskSpillover': return 'Executor memory may be too small for this cached dataset: increase executor memory or reduce its size.';
+      }
+      return undefined;
+    },
+  },
+  jobFailureRate: {
+    name: 'job failure rate',
+    tag: 'JOBS',
+    thresholdSummary: (t) => `at least ${shareLabel(t.infoRate)} of jobs failing`,
+    actionLabel: () => 'Investigate failed jobs',
+    genericRecommendation: () => 'Inspect the driver log for the failed job(s) and the stage failures that triggered them.',
+  },
+  autoscalingChurn: {
+    name: 'autoscaling churn',
+    tag: 'CHRN',
+    thresholdSummary: (t) => `over ${shareLabel(t.warningPct)} of executors living under ${t.shortLivedMs / 60000} minutes`,
+    actionLabel: () => 'Reduce autoscaling churn',
+    genericRecommendation: () => 'This looks like wasteful re-provisioning rather than normal scale-down: consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.',
+  },
+
+  configAudit: CONFIG_AUDIT_PRESENTATION,
+
+  duplicatePlanSubtree: {
+    name: 'duplicate plan subtree',
+    tag: 'PLAN',
+    thresholdSummary: () => 'the same physical plan subtree executed more than once',
+    actionLabel: () => 'Dedupe repeated subtree',
+    // isExchangeRoot isn't a Finding field, so one sentence covers both cases.
+    genericRecommendation: () => 'Check whether the repeated subtree could be computed once and reused, or cache/persist the shared computation.',
+  },
+  smallFiles: {
+    name: 'small files',
+    tag: 'PLAN',
+    thresholdSummary: (t) => `over ${t.minFiles} files averaging under ${t.maxAvgFileSizeMB} MiB`,
+    actionLabel: (f) => (f.direction === 'write' ? 'Coalesce output files' : 'Compact small files'),
+    genericRecommendation: (f) => (f.direction === 'write'
+      ? 'Repartition or coalesce before writing to raise the average file size.'
+      : 'Compact the upstream output so fewer, larger files are produced.'),
+  },
+  underBroadcast: {
+    name: 'missed broadcast join',
+    tag: 'PLAN',
+    thresholdSummary: () => 'a join below the configured size floor that skipped broadcast',
+    actionLabel: () => 'Use broadcast join',
+    genericRecommendation: () => 'This could have been a broadcast join: consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.',
+  },
+  overBroadcast: {
+    name: 'oversized broadcast join',
+    tag: 'PLAN',
+    thresholdSummary: (t) => `a broadcast over ${t.overBroadcastBytes / 1073741824} GiB`,
+    actionLabel: () => 'Fix oversized broadcast',
+    genericRecommendation: (f) => (switchAlreadyOn(f, 'spark.sql.autoBroadcastJoinThreshold')
+      ? 'Automatic broadcast is already disabled, so remove the broadcast() hint that forced it.'
+      : 'Check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.'),
+  },
+};
+
+/** The presentation row for a free-form type string (report JSON, a filter, a test double). */
+export function presentationOf(type        )                                               {
+  return (FINDING_PRESENTATION                                                              )[type];
+}

package/vendor-core/finding-tag-help.js

@@ -0,0 +1,110 @@
+// Plain-language help per finding tag: the expansion the tag stands for and a one-line description
+// of what it means. Shared by the dashboard (tag tooltips) and the
+// comparison verdict, which names finding categories by their expansion on every path.
+                
+                    
+                      
+  
+
+export const TAG_HELP                          = {
+  INCMP: {
+    expansion: 'Incomplete run',
+    description: 'This event log never recorded an application-end event, so other findings and metrics reflect only what was captured.',
+  },
+  SKEW: {
+    expansion: 'Task skew',
+    description: 'A small number of tasks take much longer than their peers.',
+  },
+  SHFL: {
+    expansion: 'Shuffle I/O',
+    description: 'Tasks are moving a large amount of intermediate data between stages.',
+  },
+  SPILL: {
+    expansion: 'Memory and disk spill',
+    description: 'Tasks are writing data out of memory, which slows execution.',
+  },
+  GC: {
+    expansion: 'Garbage collection pressure',
+    description: 'Tasks are spending an unusually large share of time reclaiming memory.',
+  },
+  COLD: {
+    expansion: 'Executor cold start',
+    description: 'New executors are taking time to become available for work.',
+  },
+  UTIL: {
+    expansion: 'Low utilization',
+    description: 'Allocated executors are idle for a large share of the application run.',
+  },
+  MEM: {
+    expansion: 'Memory utilization',
+    description: 'Executor memory or core capacity may be over- or under-provisioned.',
+  },
+  LOCAL: {
+    expansion: 'Core usage locality',
+    description: 'Tasks are running without process- or node-local data placement more than expected.',
+  },
+  HOST: {
+    expansion: 'Slow host',
+    description: 'One executor is substantially slower than its peers.',
+  },
+  FAIL: {
+    expansion: 'Failed tasks',
+    description: 'Tasks are failing often enough to affect the stage.',
+  },
+  STRAG: {
+    expansion: 'Straggler tasks',
+    description: 'A few tasks are much slower than the rest of their stage.',
+  },
+  SPEC: {
+    expansion: 'Speculation waste',
+    description: 'Speculative task attempts used a lot of executor time without confirming a genuine straggler.',
+  },
+  RETRY: {
+    expansion: 'Retry waste',
+    description: 'Repeated task attempts are consuming avoidable execution time.',
+  },
+  TINY: {
+    expansion: 'Tiny tasks',
+    description: 'Many very short tasks are adding scheduling overhead.',
+  },
+  SFAIL: {
+    expansion: 'Failed stage',
+    description: 'A stage attempt failed outright.',
+  },
+  PART: {
+    expansion: 'Partition sizing',
+    description: 'Shuffle partitions are too large, too uneven, or too few for the work.',
+  },
+  SLOW: {
+    expansion: 'Stage slowness',
+    description: 'A stage is slow overall without a more specific diagnosed cause.',
+  },
+  SHAPE: {
+    expansion: 'Stage shape',
+    description: 'The stage has an inefficient task count, output shape, or task-to-stage balance.',
+  },
+  CACHE: {
+    expansion: 'Caching opportunity',
+    description: 'A reusable dataset may benefit from being persisted between stages.',
+  },
+  CSTOR: {
+    expansion: 'Cache storage',
+    description: 'A persisted dataset is not fully cached in memory or is spilling to disk.',
+  },
+  CHRN: {
+    expansion: 'Autoscaling churn',
+    description: 'Executors are being stood up and torn down again before they can do useful work.',
+  },
+  JOBS: {
+    expansion: 'Job failure rate',
+    description: 'A large share of completed jobs did not succeed.',
+  },
+  CFG: {
+    expansion: 'Configuration audit',
+    description: 'Configuration settings may cause reliability or efficiency problems.',
+  },
+  PLAN: {
+    expansion: 'Plan advisor',
+    description: 'The SQL execution plan has a pattern worth reviewing.',
+  },
+};