sparkforensics-mcp 0.3.0 → 0.4.0

package/vendor-core/finding-types.js

@@ -0,0 +1,361 @@
+// Per-detector finding shapes. `Finding` is a union discriminated on `type`, one member per
+// emitted finding type; `FindingType` (detectors.ts) derives the same set from `DETECTORS`' `emits`
+// lists, and a compile-time check there keeps the two equal.
+//
+// Each member is split in two: `<Type>Evidence` is the public part, the fields the evidence report
+// publishes as a finding row's `evidence` (EVIDENCE_KEYS in evidence-report.ts must list every one of
+// them), and anything declared only on `<Type>Finding` is internal, read by another core module (the
+// impact estimator, mostly) and never published. Renaming or removing an evidence field is a
+// breaking change to the report and needs an EVIDENCE_SCHEMA_VERSION bump.
+                                                                      
+                                                          
+
+// The columns every finding carries, whatever its detector.
+                                         
+              
+                           
+          
+                                                                                                 
+                                                                                               
+                                                                                       
+                           
+                         
+                  
+                          
+                     
+                      
+                              
+                                  
+                                                                                              
+                                                                                      
+                                    
+ 
+
+/** One overridden threshold: the value the detector ran with, and its specification default. */
+                                 
+                                    
+                                      
+ 
+
+                                                                       
+
+// A finding whose `value` is a magnitude (a ratio, byte count, percentage, duration...). Absent on
+// an evidence caveat, which has nothing to measure.
+                                                                   
+                 
+                        
+ 
+
+// A finding whose observed value is display text (a failure reason, an audited config's current
+// value), carried in `valueText` so `value` stays numeric across every finding.
+                                                                
+                    
+                    
+ 
+
+// A failed or retried task attempt, as a stage keeps a bounded sample of them.
+                                    
+                        
+                        
+               
+                     
+                        
+                      
+                     
+                       
+ 
+
+// ── Per-stage detectors ─────────────────────────────────────────────────────
+
+                                                
+                                                                            
+
+                                     
+                                                             
+ 
+                                                                                             
+                                                                                                     
+                      
+ 
+
+                                                   
+                                                                                     
+
+                                          
+                                                                                
+ 
+                                                                                                             
+
+                                
+                                                
+ 
+                                                                               
+
+                             
+                                                         
+                    
+ 
+                                                                      
+
+// Three shapes share this type: a slow host by mean task time (no variant), a host doing most of the
+// stage's task time ('durationShare'), and an executor deviating on one dimension ('multiDim').
+                                   
+                                         
+                
+                         
+                                                                    
+                      
+                                                                           
+                      
+                                                                                                
+                        
+ 
+                                                                                        
+
+                                                         
+                                                                                                       
+
+                                      
+                          
+                   
+                             
+                                         
+ 
+// valueText is the stage's recorded failure reason.
+                                                                                              
+
+                                   
+                      
+                                
+                               
+                                                                                                
+                            
+                                    
+                           
+ 
+                                                                                        
+
+                                    
+                        
+                           
+                         
+ 
+                                                                                           
+
+                                                            
+                                                                                                                
+
+                                     
+                   
+                             
+                                          
+ 
+                                                                                             
+                                                                                        
+                    
+ 
+
+                                                    
+                                                                                        
+
+// ── App-level detectors ─────────────────────────────────────────────────────
+
+                                                         
+// valueText is always 'missing': the ApplicationEnd event the log lacks.
+                                                                                                    
+
+                                                     
+                                                                                           
+
+                                      
+                                                                                               
+                                   
+ 
+                                                                                                
+                                                                                             
+                      
+                              
+                        
+                     
+ 
+
+                                            
+                                                     
+                                                             
+                                                    
+                      
+                                                               
+                
+                                                              
+                            
+ 
+                                                                                                                  
+                                                                                               
+                                                                    
+                            
+                              
+                         
+                         
+                          
+ 
+
+                                           
+                                                                  
+                 
+                   
+                      
+                    
+                               
+                         
+                                                                     
+                            
+ 
+                                                                                                                
+
+                                       
+                            
+ 
+                                                                                                    
+
+                                           
+                                  
+ 
+                                                                                                                
+
+// A relation scanned by several SQL executions (no variant), or a join/union result several
+// executions recompute ('composite', which lists its leaf relations).
+                                             
+                        
+                   
+                 
+                                                     
+                              
+                         
+                         
+ 
+                                                                                                                      
+
+                                         
+                     
+                    
+                      
+                     
+                                                                                            
+                           
+                          
+ 
+                                                                                                          
+
+// ── Config-scope detectors ──────────────────────────────────────────────────
+
+                                      
+                   
+ 
+// valueText is the audited property's current value, or a note that it is unset or inverted.
+                                                                                              
+
+// ── SQL-scope (Plan Advisor) detectors ──────────────────────────────────────
+
+                               
+                      
+                     
+ 
+// View-layer plan-graph node ids the finding came from (plan-graph-model.ts). An array: a
+// duplicate subtree spans many nodes, and a broadcast finding can involve more than one.
+                          
+                        
+ 
+
+                                                                           
+                                                                             
+                                      
+                                
+                   
+                      
+                                
+                     
+ 
+                                            
+                                                                                                 
+
+                                                                 
+                    
+                              
+                   
+ 
+                                                                                                              
+
+                                                                     
+                          
+ 
+                                                                                                                          
+
+                                                        
+                                                                                                                       
+
+// ── Unions ──────────────────────────────────────────────────────────────────
+
+/** Public evidence fields per finding type: what a report row's `evidence` carries. */
+                                     
+                     
+                                 
+                           
+                                           
+                       
+                 
+                             
+                                       
+                                   
+                             
+                               
+                                             
+                                 
+                             
+                                       
+                               
+                                   
+                                               
+                                             
+                                     
+                                             
+                                                 
+                                         
+                                   
+                                                     
+                                 
+                                         
+                                       
+ 
+
+                     
+               
+                     
+                  
+                          
+                
+             
+                   
+                        
+                      
+                   
+                    
+                           
+                     
+                   
+                        
+                    
+                      
+                            
+                           
+                       
+                           
+                             
+                         
+                      
+                               
+                     
+                         
+                         
+
+/** The finding member for one `type`. */
+                                                                                 

package/vendor-core/findings-of-type.js

@@ -0,0 +1,11 @@
+                                                     
+
+/** The findings of one `type`, typed as that type's finding member. */
+export function findingsOfType                           (findings                    , type   )                 {
+  return findings.filter((f)                    => f.type === type);
+}
+
+/** A sql-scope finding's linked stages; undefined for every finding that keys on `stageId`. */
+export function findingStageIds(f         )                       {
+  return 'stageIds' in f ? f.stageIds : undefined;
+}

package/vendor-core/format-utils.js

@@ -1,3 +1,6 @@
+import { FINDING_PRESENTATION } from './finding-presentation.js';
+                                                              
+
 const TARGET_PARTITION_BYTES = 128 * 1024 * 1024;
 const MAX_RECOMMENDED = 8000;
 const MEANINGFUL_RATIO = 1.5;
@@ -5,11 +8,11 @@ const MEANINGFUL_RATIO = 1.5;
 export const VISIBLE_LIMIT         = 6;
 export const IMPACT_BAND_ORDER                                                  = { critical: 0, warning: 1, info: 2 };
 
-const BOTTLENECK_WIDGET                         = {
+const BOTTLENECK_WIDGET                                   = {
   skew: 'task-skew', slowHost: 'task-skew', straggler: 'task-skew',
   shuffle: 'shuffle-io', spill: 'spill', gc: 'gc-pressure', failures: 'failures',
   coldStart: 'executor-timeline', utilization: 'executor-timeline', speculationWaste: 'executor-timeline',
-};
+}                                               ;
 
 // Cross-widget stage recurrence: how many distinct board widgets (skew+straggler on one stage
 // still count as one, task-skew) flag a stage. Computed once from the full catalog each widget
@@ -30,22 +33,12 @@ export function stageWidgetFrequency(catalog
   return freq;
 }
 
-// Canonical detector type -> ALL-CAPS board tag vocabulary. Single source of truth for every
-// widget that renders a catalog entry's tag outside its own card (e.g. Bottleneck Alerts).
-// Exported so doc-sync checks can enumerate every tag without hand-duplicating this list.
-export const TYPE_TAG_MAP                         = {
-  skew: 'SKEW', shuffle: 'SHFL', spill: 'SPILL', gc: 'GC',
-  coldStart: 'COLD', utilization: 'UTIL', memoryUtilization: 'MEM',
-  cacheUtilization: 'CSTOR', coreLocality: 'LOCAL', autoscalingChurn: 'CHRN',
-  slowHost: 'HOST', failures: 'FAIL', straggler: 'STRAG',
-  retryWaste: 'RETRY', tinyTask: 'TINY', stageFailed: 'SFAIL',
-  speculationWaste: 'SPEC',
-  partitionSizing: 'PART', stageSlowness: 'SLOW', stageShape: 'SHAPE',
-  cachingOpportunity: 'CACHE', jobFailureRate: 'JOBS', configAudit: 'CFG',
-  duplicatePlanSubtree: 'PLAN', smallFiles: 'PLAN', underBroadcast: 'PLAN', overBroadcast: 'PLAN',
-  broadcastSizing: 'PLAN',
-  incompleteRun: 'INCMP',
-};
+// Finding type -> ALL-CAPS board tag, derived from FINDING_PRESENTATION. Every widget that renders
+// a catalog entry's tag outside its own card (e.g. Bottleneck Alerts) reads it through typeTag.
+// Exported so doc-sync checks can enumerate every tag. Read by free-form type string.
+export const TYPE_TAG_MAP                                   = Object.fromEntries(
+  (Object.keys(FINDING_PRESENTATION)                 ).map((type) => [type, FINDING_PRESENTATION[type].tag]),
+);
 
 export function typeTag(type        )         {
   return TYPE_TAG_MAP[type] ?? type.toUpperCase();
@@ -134,10 +127,10 @@ export function nsToMs(ns        )         {
   return ns / 1e6;
 }
 
-// Finding.value is number|string only because a couple of detectors (configAudit, stageFailed)
-// report a string; every other row formatter needs the numeric case and falls back to 0.
-export function numericValue(f                             )         {
-  return typeof f.value === 'number' ? f.value : 0;
+// A finding's magnitude, or 0 when it has none (an evidence caveat, or a text-valued finding whose
+// value is in `valueText`).
+export function numericValue(f                    )         {
+  return f.value ?? 0;
 }
 
 // Low-level "number -> display string" step shared by every widget's per-finding label resolver:
@@ -181,11 +174,11 @@ function formatChipBytes(bytes        )         {
 }
 
 /** Compact magnitude for a finding's plan-graph chip (e.g. "4.2 GB", "3.2×", "45%"), taken from the
- * finding's own `value` rendered in its `metric`'s unit. Returns null when `value` is non-numeric
- * (stageFailed/configAudit reuse it for text) or the metric isn't in FINDING_METRIC_UNIT. */
-export function formatFindingMagnitude(finding                                              )                {
+ * finding's own `value` rendered in its `metric`'s unit. Returns null when there is no `value` (a
+ * text-valued finding carries `valueText` instead) or the metric isn't in FINDING_METRIC_UNIT. */
+export function formatFindingMagnitude(finding                                     )                {
   const unit = finding.metric ? FINDING_METRIC_UNIT[finding.metric] : undefined;
-  if (!unit || typeof finding.value !== 'number') return null;
+  if (!unit || finding.value == null) return null;
   switch (unit) {
     case 'bytes': return formatChipBytes(finding.value);
     case 'minutes': return formatDuration(finding.value * 60000);
@@ -201,7 +194,7 @@ export function formatFindingMagnitude(finding
  * recovery is claimed, "~<time>" (the optimistic `high` bound, matching formatImpactEstimateCompact),
  * joined by " · " (e.g. "4.2 GB · ~38.0s"). null when neither part is available. */
 export function formatFindingChipDetail(
-  finding                                                                                                                     ,
+  finding                                                                                                            ,
 )                {
   const magnitude = formatFindingMagnitude(finding);
   const wallClock = finding.impactEstimate?.wallClock;
@@ -239,3 +232,75 @@ export function buildHistogram(values          , bins        )
   const labels = counts.map((_, i) => Math.round(min + i * binSize));
   return { labels, data: counts };
 }
+
+// Unit formatting for savings and resource figures, shared by the interpretation and by
+// renderers that total a filtered selection themselves (the recommendation rollup).
+export function fmtMs(ms        )         {
+  return ms === 0 ? '0s' : formatDuration(ms);
+}
+
+export function formatWallClockRange(low        , high        )         {
+  const lowText = fmtMs(low);
+  const highText = fmtMs(high);
+  // Compare the formatted strings, not the raw ms values: formatDuration
+  // floors to whole seconds (or minutes+seconds) once a value hits 60s, so
+  // two endpoints that differ by under a bucket's precision (e.g. 140900ms
+  // vs 141200ms, both "2m 21s") would still fail a raw low === high check
+  // and render as a degenerate "2m 21s-2m 21s" range.
+  if (lowText === highText) return highText;
+  return `${lowText}-${highText}`;
+}
+
+const MS_PER_HOUR = 3_600_000;
+const MB_SECONDS_PER_GB_HOUR = 1024 * 3600;
+/** Below this many hours a figure keeps its small unit, so "0.0 GB-h" never
+ * hides a real but small amount. */
+const MIN_HOURS_SHOWN = 0.1;
+
+const oneDecimal = (value        )         =>
+  (Math.round(value * 10) / 10).toLocaleString('en-US', { minimumFractionDigits: 1, maximumFractionDigits: 1 });
+
+export function formatRawWaste(rawWaste                )         {
+  const rounded = Math.round(rawWaste.value * 10) / 10;
+  switch (rawWaste.unit) {
+    case 'bytes':
+      return formatBytes(rawWaste.value);
+    case 'ms':
+      return fmtMs(rawWaste.value);
+    case 'mbSeconds': {
+      // A whole run's idle memory reaches millions of MB-seconds; GB-hours
+      // keeps it a number a reader can compare.
+      const gbHours = rawWaste.value / MB_SECONDS_PER_GB_HOUR;
+      return gbHours >= MIN_HOURS_SHOWN ? `${oneDecimal(gbHours)} GB-h` : `${rounded.toLocaleString('en-US')} MB-s`;
+    }
+    case 'coreHours':
+      return `${rounded.toFixed(1)} core-h`;
+    case 'coreMs': {
+      const coreHours = rawWaste.value / MS_PER_HOUR;
+      return coreHours >= MIN_HOURS_SHOWN ? `${oneDecimal(coreHours)} core-h` : `${oneDecimal(rawWaste.value / 1000)} core-s`;
+    }
+    default:
+      return String(rawWaste.value);
+  }
+}
+
+// formatRawWaste/fmtMs round to a fixed precision (one decimal place, or
+// whole milliseconds below 1s), which can collapse a small but genuinely
+// nonzero value down to a formatted string that reads exactly like zero
+// (0.04 coreHours -> "0.0 core-h"). Checking the raw value against 0 misses
+// this; checking the *formatted* text against zero's own formatted text
+// catches it regardless of unit. The fractional group must consume the
+// entire decimal part (all zeros) before the terminator: otherwise a real
+// value like "0.5 core-h" leaves the "." unconsumed, and the terminator
+// char class excludes ".", so it correctly fails to match.
+export function readsAsZero(formatted        )          {
+  return /^0(\.0+)?(?:[^0-9.]|$)/.test(formatted);
+}
+
+/** Whole cores from 10 up; one decimal below, so a small run's peak never
+ * rounds down to "0 cores". */
+export function formatCores(cores        )         {
+  return cores >= 10 ? String(Math.round(cores)) : cores.toFixed(1).replace(/\.0$/, '');
+}
+
+export const MS_PER_CORE_HOUR         = 3.6e6;

package/vendor-core/html-export.js

@@ -0,0 +1,51 @@
+import { auditConfig } from './analyzer.js';
+import { buildExportRunData, CORE_VERSION,                                           } from './export-data.js';
+import { redactRunModel } from './redact.js';
+import { interpretRun } from './run-interpretation.js';
+import { gzipSync, strToU8 } from './vendor/fflate.js';
+                                                    
+
+// Shared by the two producers of the self-contained HTML dashboard: the CLI's
+// --export-html (writeHtmlExport, which writes a data.js next to the template)
+// and the dashboard's "Download HTML" item (EvidenceExport.tsx, which inlines
+// the same statement into one downloaded file). Both encode through
+// encodeRunPayload here and wrap it with runPayloadScript (run-payload.ts); the
+// export app's decodeRunPayload (src/export/hydrate-store.ts) reverses it.
+
+/** The run data an HTML export carries: the serialized model, the config
+ * audit, and the whole interpretation layer (verdict, coverage, formatted
+ * savings, run shape) computed here, so the bundle that opens the file renders
+ * this core's conclusions instead of deriving its own. With `redact`, the model
+ * and findings are pseudonymized first and the redacted run is interpreted, so
+ * text the interpretation truncates (Spark's failure reason) can never keep
+ * part of an identifier the redactor would no longer recognize. */
+export function buildHtmlExportData(
+  appModel          ,
+  catalog           ,
+  skippedLines        ,
+  { redact, buildId, producer }                                                        ,
+)                {
+  const raw = { appModel, catalog, configFindings: auditConfig(appModel.app) };
+  const run = redact ? redactRunModel(raw.appModel, raw.catalog, raw.configFindings) : raw;
+  const interpretation = interpretRun(run.appModel, run.catalog, run.configFindings);
+  const provenance                   = { coreVersion: CORE_VERSION, buildId, producer };
+  return buildExportRunData(run.appModel, run.catalog, run.configFindings, skippedLines, interpretation, provenance);
+}
+
+// String.fromCharCode spreads its arguments onto the stack; a multi-MB payload
+// in one call overflows it, so convert in slices.
+const BASE64_CHUNK_BYTES = 0x8000;
+
+function bytesToBase64(bytes            )         {
+  let binary = '';
+  for (let i = 0; i < bytes.length; i += BASE64_CHUNK_BYTES) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + BASE64_CHUNK_BYTES));
+  }
+  return btoa(binary);
+}
+
+/** gzip + base64 of the JSON payload, runnable in the browser (fflate, no
+ * node:zlib). strToU8 encodes UTF-8, matching decodeRunPayload's decode. */
+export function encodeRunPayload(data               )         {
+  return bytesToBase64(gzipSync(strToU8(JSON.stringify(data))));
+}

package/vendor-core/impact-band.js

@@ -1,12 +1,19 @@
                                                         
-import { STRAGGLER_FLOOR_PCT_WARN, STRAGGLER_FLOOR_PCT_CRIT } from './detectors.js';
 
-// Reused from straggler's own thresholds (marked NOT SOURCED: unvalidated there): apply the same
-// accepted noise floor globally rather than inventing a second cutoff. Imported so they can't drift.
-const IMPACT_FLOOR_PCT_WARN = STRAGGLER_FLOOR_PCT_WARN;
-const IMPACT_FLOOR_PCT_CRIT = STRAGGLER_FLOOR_PCT_CRIT;
+// The run-wide noise floor, as a share of the run's duration, that grades every wall-clock
+// estimate (NOT SOURCED: our own, unvalidated). skew and straggler default their firing floors to
+// these same figures (detectors.ts), so a finding they admit grades at least warning here.
+export const IMPACT_FLOOR_PCT_WARN = 0.005;
+export const IMPACT_FLOOR_PCT_CRIT = 0.02;
 
-function appDurationMs(app                     )                {
+/** The share-of-run floors one finding grades against. */
+                                                                      
+
+const DEFAULT_FLOORS                   = { warnPct: IMPACT_FLOOR_PCT_WARN, critPct: IMPACT_FLOOR_PCT_CRIT };
+
+/** The run's wall-clock duration, or null when unknown or non-positive: the denominator of both
+ * this band and the detectors' runtime floors. */
+export function appDurationMs(app                                                               )                {
   if (app?.startTime == null || app?.endTime == null) return null;
   const durationMs = app.endTime - app.startTime;
   return durationMs > 0 ? durationMs : null;
@@ -23,8 +30,13 @@ function appDurationMs(app                     )                {
  * figure), a deliberate split from view/impact-sort.ts's 'impact' sort, which ranks by .low so an
  * optimistic-but-contended finding never outranks a smaller certain one. Same range, two fields for
  * two questions.
+ *
+ * `floorsFor` supplies a finding type's floors when a tuned run moved them (skew's and straggler's
+ * own floorPctWarn/floorPctCrit); omitted, or for any type it returns nothing, the defaults above.
  */
-export function deriveImpactBand(findings           , app                     )            {
+export function deriveImpactBand(
+  findings           , app                     , floorsFor                                            ,
+)            {
   const durationMs = appDurationMs(app);
   if (durationMs == null) return findings;
   for (const finding of findings) {
@@ -38,7 +50,8 @@ export function deriveImpactBand(findings           , app                     )
     const recoverableMs = finding.impactEstimate?.wallClock?.high;
     if (recoverableMs == null) continue;
     const pct = recoverableMs / durationMs;
-    finding.impactBand = pct >= IMPACT_FLOOR_PCT_CRIT ? 'critical' : pct >= IMPACT_FLOOR_PCT_WARN ? 'warning' : 'info';
+    const { warnPct, critPct } = floorsFor?.(finding.type) ?? DEFAULT_FLOORS;
+    finding.impactBand = pct >= critPct ? 'critical' : pct >= warnPct ? 'warning' : 'info';
   }
   return findings;
 }