sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/export-data.js

@@ -2,11 +2,33 @@
                                                             
                                                                                 
                     
+                                                                 
 
-export const EXPORT_DATA_SCHEMA_VERSION = 1;
+/** Version 2 carries the interpretation layer (`interpretation`) and `provenance`. Version 3 types
+ * findings per detector: a text figure (stageFailed, configAudit, incompleteRun) moved from `value`
+ * to `valueText`, and Plan Advisor findings always carry `planNodeIds`. The export bundle renders
+ * only the version it was built for and refuses any other, since an older payload lacks the
+ * conclusions it would otherwise have to derive, or carries them in fields this bundle no longer reads. */
+export const EXPORT_DATA_SCHEMA_VERSION = 3;
+
+/** @sparkforensics/core's own version (packages/core/package.json, which a test keeps in sync).
+ * A constant, not a JSON import, because the vendored copies in cli/mcp/server ship without
+ * that package.json. */
+export const CORE_VERSION = '0.1.0';
+
+/** What produced an export, shown in the exported dashboard's footer. */
+                                   
+                      
+                                                                                              
+                                           
+                  
+                                                                            
+                   
+ 
 
                                 
                         
+                               
                            
                   
               
@@ -17,23 +39,28 @@ export const EXPORT_DATA_SCHEMA_VERSION = 1;
                      
                             
                        
+                                    
  
 
 /** Converts an in-memory AppModel (Map-based, as collectRun()/the browser
- * worker produce it) plus the CLI's already-computed catalog/configFindings
- * into a plain, JSON-serializable object for the HTML export's data.js.
+ * worker produce it) plus its catalog/configFindings and their interpretation
+ * (`interpretRun`, which indexes them in this same catalog-then-config order) into a plain, JSON-serializable object for the HTML export's data.js.
  * Maps become arrays; main-export.tsx rebuilds them client-side keyed the
  * same way the store already expects (Stage.id, Job.id,
- * SqlExecution.id). Deliberately excludes raw per-task TaskData, which is far
+ * SqlExecution.id). Maps and Sets nested deeper keep their type through
+ * `reviveExportCollections`. Deliberately excludes raw per-task TaskData, which is far
  * too large to inline; task-detail widgets degrade gracefully in export mode. */
 export function buildExportRunData(
   appModel          ,
   catalog           ,
   configFindings           ,
   skippedLines        ,
+  interpretation                   ,
+  provenance                  ,
 )                {
-  return {
+  return encodeCollections({
     schemaVersion: EXPORT_DATA_SCHEMA_VERSION,
+    provenance,
     app: appModel.app,
     stages: [...appModel.stages.values()],
     jobs: [...appModel.jobs.values()],
@@ -44,5 +71,51 @@ export function buildExportRunData(
     catalog,
     configFindings,
     skippedLines,
-  };
+    interpretation,
+  })                 ;
+}
+
+// Maps and Sets nested inside the payload (`app.rddInfo`, each stage's
+// `executorMetrics`) would serialize to `{}` under plain JSON, and a widget
+// calling `.values()` on one then throws. They travel as tagged plain objects
+// instead, which survive JSON and redaction's deep copy alike;
+// `reviveExportCollections` turns them back on load.
+const MAP_TAG = '__sparkforensicsMap';
+const SET_TAG = '__sparkforensicsSet';
+
+export function encodeCollections(value         )          {
+  if (value instanceof Map) return { [MAP_TAG]: [...value].map(([k, v]) => [encodeCollections(k), encodeCollections(v)]) };
+  if (value instanceof Set) return { [SET_TAG]: [...value].map(encodeCollections) };
+  if (Array.isArray(value)) return value.map(encodeCollections);
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, encodeCollections(v)]));
+  }
+  return value;
+}
+
+/** The in-memory counterpart of `reviveExportCollections`: rebuilds every
+ * tagged Map and Set in a tree `encodeCollections` produced, without a JSON
+ * round trip. */
+export function decodeCollections(value         )          {
+  if (Array.isArray(value)) return value.map(decodeCollections);
+  if (value && typeof value === 'object') {
+    const record = value                           ;
+    if (Array.isArray(record[MAP_TAG])) {
+      return new Map((record[MAP_TAG]                        ).map(([k, v]) => [decodeCollections(k), decodeCollections(v)]));
+    }
+    if (Array.isArray(record[SET_TAG])) return new Set((record[SET_TAG]             ).map(decodeCollections));
+    return Object.fromEntries(Object.entries(record).map(([k, v]) => [k, decodeCollections(v)]));
+  }
+  return value;
+}
+
+/** `JSON.parse` reviver for the export payload: rebuilds every Map and Set
+ * `buildExportRunData` tagged. */
+export function reviveExportCollections(_key        , value         )          {
+  if (value && typeof value === 'object' && !Array.isArray(value)) {
+    const record = value                           ;
+    if (Array.isArray(record[MAP_TAG])) return new Map(record[MAP_TAG]                        );
+    if (Array.isArray(record[SET_TAG])) return new Set(record[SET_TAG]             );
+  }
+  return value;
 }

package/vendor-core/finding-action-label.js

@@ -1,91 +1,12 @@
+import { findingName } from './finding-names.js';
+import { presentationOf } from './finding-presentation.js';
                                           
 
-/** A short, imperative action label for a finding's row. Keyed off finding.type plus whichever
- * discriminant field that detector uses (rule/direction/variant/property). Core-safe (no view
- * import) so the dashboard's wrapper and evidence-report.ts share the same switch. Returns
- * undefined for any combination this switch doesn't cover; the caller decides the fallback. */
-export function coreFindingActionLabel(finding         )                     {
-  switch (finding.type) {
-    case 'skew':
-      return 'Fix task skew';
-    case 'stageShape':
-      switch (finding.rule) {
-        case 'lowParallelism': return 'Increase parallelism';
-        case 'dataExplosion': return 'Check for exploding join';
-        case 'taskStageSkew': return 'Fix straggler task';
-      }
-      break;
-    case 'shuffle':
-      return 'Reduce shuffle size';
-    case 'partitionSizing':
-      switch (finding.rule) {
-        case 'shufflePartitionSkew': return 'Fix skewed partition';
-        case 'lowShuffleParallelism': return 'Add shuffle partitions';
-        case 'maxPartitionTooBig': return 'Repartition oversized data';
-      }
-      break;
-    case 'spill':
-      return 'Reduce spill';
-    case 'gc':
-      return finding.direction === 'low' ? 'Right-size executor memory' : 'Reduce GC pressure';
-    case 'slowHost':
-      if (finding.variant === 'durationShare') return 'Fix data locality';
-      if (finding.variant === 'multiDim') return 'Investigate degraded executor';
-      return 'Check slow host';
-    case 'stageSlowness':
-      return 'Profile slow stage';
-    case 'stageFailed':
-      return 'Inspect stage failure';
-    case 'failures':
-      return 'Investigate task failures';
-    case 'straggler':
-      return 'Fix stragglers';
-    case 'speculationWaste':
-      return 'Tune speculation settings';
-    case 'retryWaste':
-      return 'Investigate retry cause';
-    case 'tinyTask':
-      return 'Coalesce small tasks';
-    case 'coldStart':
-      return 'Pre-warm cluster';
-    case 'utilization':
-      return 'Reduce cluster size';
-    case 'memoryUtilization':
-      switch (finding.variant) {
-        case 'idleCores': return 'Reduce idle cores';
-        case 'wasteModel': return 'Right-size executor memory';
-        case 'memoryBand':
-          if (finding.dataUnavailable) return 'Enable memory metrics';
-          return finding.rule === 'heapNearCapacity' ? 'Increase executor memory' : 'Reduce executor memory';
-      }
-      break;
-    case 'cacheUtilization':
-      return 'Increase cache memory';
-    case 'coreLocality':
-      return 'Fix data locality';
-    case 'autoscalingChurn':
-      return 'Reduce autoscaling churn';
-    case 'cachingOpportunity':
-      return finding.variant === 'composite' ? 'Cache repeated result' : 'Cache shared table';
-    case 'jobFailureRate':
-      return 'Investigate failed jobs';
-    case 'configAudit':
-      switch (finding.property) {
-        case 'spark.shuffle.service.enabled': return 'Enable shuffle service';
-        case 'spark.dynamicAllocation.minExecutors': return 'Fix autoscaling bounds';
-        case 'spark.dynamicAllocation.maxExecutors': return 'Set max executors';
-        case 'spark.serializer': return 'Switch to Kryo';
-        case 'spark.executor.memoryOverhead': return 'Raise memory overhead';
-      }
-      break;
-    case 'duplicatePlanSubtree':
-      return 'Dedupe repeated subtree';
-    case 'smallFiles':
-      return finding.direction === 'write' ? 'Coalesce output files' : 'Compact small files';
-    case 'underBroadcast':
-      return 'Use broadcast join';
-    case 'overBroadcast':
-      return 'Fix oversized broadcast';
-  }
-  return undefined;
+/** A short, imperative action label for a finding's row ("Reduce shuffle size"), from its type's
+ * FINDING_PRESENTATION row. A (type, discriminant) combination the row doesn't recognize falls
+ * back to the type's name, and an unknown type to its raw string. The dashboard, the run verdict
+ * and the evidence report all call this one function, so a finding shows the same label on every
+ * path. */
+export function findingActionLabel(finding         )         {
+  return presentationOf(finding.type)?.actionLabel(finding) ?? findingName(finding.type);
 }

package/vendor-core/finding-filter-predicate.js

@@ -18,6 +18,15 @@ function has   (m               , value   )          {
                                         
  
 
+/** The one stage a finding is about: its own `stageId`, or the only entry of a sql-scope
+ * finding's `stageIds`. Null for an app-level, config or multi-stage finding. The same rule the
+ * run verdict groups steps by and the stage dialog lists a stage's findings by. */
+export function singleStageId(row                                                                  )                {
+  if (typeof row.stageId === 'number') return row.stageId;
+  if (row.stageIds && row.stageIds.length === 1) return row.stageIds[0];
+  return null;
+}
+
 export function matchesFindingFilterCriteria(
   row                                                               ,
   criteria                       ,

package/vendor-core/finding-generic-recommendation.js

@@ -1,112 +1,14 @@
+import { presentationOf } from './finding-presentation.js';
                                           
 
 /** A generic, type-level recommendation sentence for a finding: the shape of the fix, with no
- * instance data (numbers, stage ids, host names, file counts, config values). Mirrors
- * coreFindingActionLabel's (type, discriminant) switch (same fields: rule/direction/variant/
- * property, plus cacheUtilization's variant and memoryUtilization's dataUnavailable), but covers
- * every finding type, not just the ones with a distinct action label.
+ * instance data (numbers, stage ids, host names, file counts, config values), from its type's
+ * FINDING_PRESENTATION row.
  *
  * Used for a multi-finding group's muted description line (TypeGroupRow in FixTheseFirst.tsx),
  * where the highest-impact member's own `recommendation` (real numbers, one stage) would
- * misrepresent a summed-impact trailing stat covering every member. Returns undefined where a
- * detector's real branch key isn't exposed as a Finding field (spill's skew/volume
- * classification, tinyTask's shuffle-vs-no-shuffle fix, duplicatePlanSubtree's isExchangeRoot),
- * so those combine into one sentence covering both cases, or for any (type, discriminant)
- * combination this switch doesn't recognize; the caller shows no muted line rather than guess. */
+ * misrepresent a summed-impact trailing stat covering every member. Undefined where the row has no
+ * sentence for the finding; the caller shows no muted line rather than guess. */
 export function coreFindingGenericRecommendation(finding         )                     {
-  switch (finding.type) {
-    case 'skew':
-      return 'For join-driven skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key to reduce task skew.';
-    case 'stageShape':
-      switch (finding.rule) {
-        case 'lowParallelism': return 'Too few tasks run relative to the cores available, leaving cluster capacity idle: repartition to use more of it.';
-        case 'dataExplosion': return 'Output volume far exceeds input volume: check for an exploding join or a cross product.';
-        case 'taskStageSkew': return 'A single straggler task gates the whole stage\'s wall-clock duration.';
-      }
-      break;
-    case 'shuffle':
-      return 'Consider increasing spark.sql.shuffle.partitions or adding a broadcast join to shrink the shuffle.';
-    case 'partitionSizing':
-      switch (finding.rule) {
-        case 'shufflePartitionSkew': return 'For join skew, enable AQE skew-join handling (spark.sql.adaptive.skewJoin.enabled); otherwise salt the key or repartition on a better key.';
-        case 'lowShuffleParallelism': return 'Raise spark.sql.shuffle.partitions so each partition is smaller.';
-        case 'maxPartitionTooBig': return 'Repartition to break up the oversized partition before this stage.';
-      }
-      break;
-    case 'spill':
-      return 'If the spill is skew-driven, fix task skew first: adding memory will not help. Otherwise raise spark.sql.shuffle.partitions or increase executor memory.';
-    case 'gc':
-      return finding.direction === 'low'
-        ? 'Memory may be over-provisioned here: consider reducing spark.executor.memory for cost savings.'
-        : 'Reduce object creation, use primitive types, avoid UDFs, or increase executor memory to cut GC time.';
-    case 'slowHost':
-      if (finding.variant === 'durationShare') return 'Check for data locality or partition assignment skewing work onto one node.';
-      if (finding.variant === 'multiDim') return 'Investigate uneven partition assignment or a degraded executor.';
-      return 'Check what this host was running: it may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable spark.speculation to relaunch a lagging task automatically.';
-    case 'stageSlowness':
-      return 'Often a partition-count problem: raise parallelism via spark.sql.shuffle.partitions or spark.default.parallelism, or check for a large per-task data volume driving heavy shuffle and spill.';
-    case 'stageFailed':
-      return 'Inspect the driver log for the failure reason and the job that triggered it.';
-    case 'failures':
-      return 'Investigate driver logs for executor instability or data-driven errors.';
-    case 'straggler':
-      return 'Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue. If a skewed key is the real cause, that is a candidate for AQE\'s skew-join handling.';
-    case 'speculationWaste':
-      return 'If task durations are naturally variable rather than genuine stragglers, consider tuning spark.speculation.multiplier/quantile.';
-    case 'retryWaste':
-      return 'Investigate executor loss or fetch failures behind the retried attempts.';
-    case 'tinyTask':
-      return 'Scheduler overhead may dominate: lower spark.sql.shuffle.partitions, or coalesce down to fewer, larger tasks.';
-    case 'coldStart':
-      return 'Keep a warm pool of idle executors, or if using dynamic allocation, raise the minimum/initial executor count so it does not scale up from zero.';
-    case 'utilization':
-      return 'Consider reducing cluster size or enabling dynamic allocation.';
-    case 'memoryUtilization':
-      switch (finding.variant) {
-        case 'idleCores': return 'Reduce cluster size or enable dynamic allocation.';
-        case 'wasteModel': return 'Review spark.executor.memory and executor count.';
-        case 'memoryBand':
-          if (finding.dataUnavailable) break;
-          return finding.rule === 'heapNearCapacity'
-            ? 'Memory may be too small: raise spark.executor.memory to avoid OOM/spill.'
-            : 'Memory may be over-provisioned: consider reducing spark.executor.memory for cost savings.';
-      }
-      break;
-    case 'cacheUtilization':
-      switch (finding.variant) {
-        case 'partialCache': return 'Increase executor memory or reduce the cached dataset size so more of it stays cached.';
-        case 'diskSpillover': return 'Executor memory may be too small for this cached dataset: increase executor memory or reduce its size.';
-      }
-      break;
-    case 'coreLocality':
-      return 'Check spark.locality.wait settings and executor/data colocation.';
-    case 'autoscalingChurn':
-      return 'This looks like wasteful re-provisioning rather than normal scale-down: consider raising spark.dynamicAllocation.executorIdleTimeout or widening the minExecutors/maxExecutors bounds to reduce flapping.';
-    case 'cachingOpportunity':
-      return finding.variant === 'composite'
-        ? 'Cache or persist the repeated join/union result so it is computed once instead of recomputed per query.'
-        : 'Cache the shared DataFrame, or broadcast it if it is a small join lookup.';
-    case 'jobFailureRate':
-      return 'Inspect the driver log for the failed job(s) and the stage failures that triggered them.';
-    case 'configAudit':
-      switch (finding.property) {
-        case 'spark.shuffle.service.enabled': return 'Set spark.shuffle.service.enabled=true so shuffle data survives executor removal.';
-        case 'spark.dynamicAllocation.minExecutors': return 'Set the minimum executor bound at or below the maximum.';
-        case 'spark.dynamicAllocation.maxExecutors': return 'Set spark.dynamicAllocation.maxExecutors to cap cluster growth.';
-        case 'spark.serializer': return 'Consider spark.serializer=org.apache.spark.serializer.KryoSerializer for faster, smaller buffers.';
-        case 'spark.executor.memoryOverhead': return 'Raise executor memoryOverhead above Spark\'s default floor to avoid off-heap OOM-kills.';
-      }
-      break;
-    case 'duplicatePlanSubtree':
-      return 'Check whether the repeated subtree could be computed once and reused, or cache/persist the shared computation.';
-    case 'smallFiles':
-      return finding.direction === 'write'
-        ? 'Repartition or coalesce before writing to raise the average file size.'
-        : 'Compact the upstream output so fewer, larger files are produced.';
-    case 'underBroadcast':
-      return 'This could have been a broadcast join: consider a broadcast() hint or raising spark.sql.autoBroadcastJoinThreshold.';
-    case 'overBroadcast':
-      return 'Check for a misapplied broadcast hint or a misconfigured spark.sql.autoBroadcastJoinThreshold.';
-  }
-  return undefined;
+  return presentationOf(finding.type)?.genericRecommendation(finding);
 }

package/vendor-core/finding-names.js

@@ -1,51 +1,27 @@
-// Canonical detector `type` -> human-readable label (single source of truth). detector-registry
-// imports this (web uses it lowercase); evidence-report.ts Title Cases it for CLI/MCP names.
-//
-// broadcastSizing is the DETECTORS-level type but no real Finding carries it (the detector pushes
-// underBroadcast/overBroadcast). Kept as a dead key so the DETECTORS-type completeness check finds it.
-export const FINDING_NAMES                         = {
-  incompleteRun: 'incomplete run',
-
-  skew: 'task skew',
-  stageShape: 'stage shape',
-  tinyTask: 'tiny tasks',
-
-  shuffle: 'shuffle I/O',
-  partitionSizing: 'partition sizing',
-
-  spill: 'spill',
-
-  gc: 'GC pressure',
-
-  stageFailed: 'failed stage',
-  failures: 'failed tasks',
-  retryWaste: 'retry waste',
-
-  slowHost: 'slow executor host',
-  stageSlowness: 'slow stage',
-  straggler: 'straggling task',
-  speculationWaste: 'speculation waste',
-  coldStart: 'cold start',
-
-  memoryUtilization: 'memory utilization',
-  utilization: 'executor utilization',
-  coreLocality: 'core locality',
-  cachingOpportunity: 'caching opportunity',
-  cacheUtilization: 'cache utilization',
-  jobFailureRate: 'job failure rate',
-  autoscalingChurn: 'autoscaling churn',
-
-  configAudit: 'config audit',
-
-  duplicatePlanSubtree: 'duplicate plan subtree',
-  smallFiles: 'small files',
-  broadcastSizing: 'broadcast sizing',
-  underBroadcast: 'missed broadcast join',
-  overBroadcast: 'oversized broadcast join',
-};
+import { FINDING_PRESENTATION, presentationOf } from './finding-presentation.js';
+                                                       
+
+// Finding `type` -> human-readable label, derived from FINDING_PRESENTATION. The web uses it
+// lowercase; evidence-report.ts Title Cases it for CLI/MCP names. Declared string-indexed for
+// lookups by a type read from report JSON or a filter.
+export const FINDING_NAMES                                   = Object.fromEntries(
+  (Object.keys(FINDING_PRESENTATION)                 ).map((type) => [type, FINDING_PRESENTATION[type].name]),
+);
+
+/** A finding type's name, or the raw type string for a type with no presentation row. */
+export function findingName(type        )         {
+  return presentationOf(type)?.name ?? type;
+}
 
 // Capitalizes the first letter of each word, leaving other characters untouched so acronyms
 // ('GC', 'I/O') survive.
 export function titleCase(label        )         {
   return label.replace(/\b\w/g, (c) => c.toUpperCase());
 }
+
+/** The finding's own recommendation, or its type's name when it has none. */
+export function recommendationText(finding         )         {
+  const text = typeof finding.recommendation === 'string' ? finding.recommendation.trim() : '';
+  if (text) return text;
+  return findingName(finding.type);
+}