sparkforensics-mcp 0.3.0 → 0.5.0

package/vendor-core/redact.js

@@ -7,14 +7,14 @@
 // Deterministic (sorted assignment), idempotent (pseudonyms map to themselves),
 // and non-mutating (returns a fresh, deep-copied tree).
 
-                                                      
-import { redactTaskFailureGroup,                        } from './task-failure.js';
+import { decodeCollections, encodeCollections,                    } from './export-data.js';
+                                                    
+import { REDACTED_TEXT, redactTaskFailureGroup,                        } from './task-failure.js';
 
 // Host / IP identifier patterns. Used to enumerate host names that surface only
-// inside free text: recommendation strings, `stageFailed`'s failure-reason
-// value, SQL relation/node names, never as a structured `host` field, so
-// redaction reaches those residuals too. Pseudonyms (`host-1`) match neither
-// pattern, keeping the scan idempotent.
+// inside free text: recommendation strings, SQL relation/node names, never as a
+// structured `host` field, so redaction reaches those residuals too. Pseudonyms
+// (`host-1`) match neither pattern, keeping the scan idempotent.
 const HOST_PATTERNS = [
   // EC2-style ip-10-1-2-3 with an optional dotted domain (ip-10-1-2-3.ec2.internal).
   // Each domain label must start with an alphanumeric, so a trailing sentence
@@ -35,7 +35,7 @@ const APP_ID_PATTERNS = [/\bapplication_\d{10,}_\d+\b/g];
 // Walk every string in the tree once, collecting matches for each `{ patterns,
 // out }` sink. One shared traversal for every token kind (instead of one
 // traversal per kind) keeps redactComparison's dual host+app-id scan the same
-// cost as the single-kind scan redactReport/redactAppIdentity already do.
+// cost as the single-kind scan redactReport already does.
 function scanTokens(node         , sinks                                                 )       {
   if (typeof node === 'string') {
     for (const { patterns, out } of sinks) {
@@ -55,11 +55,6 @@ function scanTokens(node         , sinks
   }
 }
 
-// Walk every string in the tree, collecting host/IP tokens into `hosts`.
-function scanHostTokens(node         , hosts             )       {
-  scanTokens(node, [{ patterns: HOST_PATTERNS, out: hosts }]);
-}
-
 // Recursively collects every string value found under a key literally named
 // `host`, anywhere in the tree. Host names surface at several depths, a
 // finding's own `host`, `evidence.host`, and now
@@ -99,12 +94,37 @@ function redactFailureGroups   (node   )    {
   return node;
 }
 
+// A stage's failure reason is Spark's free-text message and can carry file paths and data values
+// too, so it is replaced outright, like a failure group's message: a `stageFailed` finding's
+// `valueText`, the `stageFailureReason` of a stage record, a job's `exception` (the run outcome's
+// fallback reason), and the report's `failureReason` copy. Walking by key name, like
+// collectHostFields, needs no path list. Returns a fresh tree.
+const REASON_KEYS = new Set(['stageFailureReason', 'exception', 'failureReason']);
+function redactStageFailureReasons   (node   )    {
+  if (Array.isArray(node)) return node.map((n) => redactStageFailureReasons(n))     ;
+  if (node && typeof node === 'object') {
+    const isStageFailed = (node                      ).type === 'stageFailed';
+    const out                          = {};
+    for (const [k, v] of Object.entries(node)) {
+      const isReason = REASON_KEYS.has(k) || (isStageFailed && k === 'valueText');
+      out[k] = isReason && typeof v === 'string' ? REDACTED_TEXT : redactStageFailureReasons(v);
+    }
+    return out     ;
+  }
+  return node;
+}
+
+// Both passes for free text no host/app-id pattern recognizes.
+function redactFailureText   (node   )    {
+  return redactStageFailureReasons(redactFailureGroups(node));
+}
+
 // Spark config keys ending in `host`/`hostname` (e.g. spark.driver.host,
 // spark.yarn.am.hostname) carry plain FQDN host names that neither
 // HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
 // walk catches (the literal key is the dotted Spark property name, never
 // `host` itself). app.config is a flat Record<string, string> unique to
-// redactExportData: no other redact* export ships a raw Spark config dict.
+// redactRunModel: no other redact* export ships a raw Spark config dict.
 // Known gap: a hostname value under a differently-named key isn't caught by
 // this suffix check. Confirmed against a real cluster config: spark.master,
 // spark.yarn.historyServer.address, and the plural YARN proxy/HA keys
@@ -126,7 +146,7 @@ function collectConfigHostValues(config                                    , hos
                             
                                                                           
                                                                               
-                                             
+                                                                   
                                                                                
                                                                               
                  
@@ -192,28 +212,19 @@ function applyReplacements   (node   , ids
   return deepReplace(node, merged)     ;
 }
 
-export function redactReport                            (input   )    {
-  const report = redactFailureGroups(input);
-  const { appIds, hosts } = collectIds(report);
-  return applyReplacements(report, { appIds, hosts });
+// The app name identifies a job as much as its id, so a redacted name becomes the id's
+// pseudonym, as list_runs does. Only the structured field is replaced, never substrings: a
+// short name ("t", "etl") would otherwise corrupt every string it happens to occur in.
+function withRedactedName                                                        (app   )    {
+  return app.name == null ? app : { ...app, name: app.id ?? null };
 }
 
-// Narrow counterpart to redactReport(), for getRunSummary()'s standalone app
-// object (no findings tree to walk). There's exactly one app id here, so no
-// Set/Map/sort is needed for it; name/sparkVersion still go through the
-// shared host/IP scan-and-replace since either can carry a host token as
-// free text.
-export function redactAppIdentity(
-  app                                                                         ,
-)                                                                          {
-  const hosts = new Set        ();
-  scanHostTokens(app.name, hosts);
-  scanHostTokens(app.sparkVersion, hosts);
-  return {
-    id: typeof app.id === 'string' && app.id.length > 0 ? 'app-1' : app.id,
-    name: applyReplacements(app.name, { hosts }),
-    sparkVersion: applyReplacements(app.sparkVersion, { hosts }),
-  };
+export function redactReport                            (input   )    {
+  const report = redactFailureText(input);
+  const { appIds, hosts } = collectIds(report);
+  const out = applyReplacements(report, { appIds, hosts });
+  const app = out.summary?.app;
+  return app ? { ...out, summary: { ...out.summary, app: withRedactedName(app) } } : out;
 }
 
 // Run-comparison counterpart: no single app-id *field* to pseudonymize
@@ -237,8 +248,10 @@ export function redactComparison   (comparison   )    {
 // this also walks executors.added/removed for their literal `host` field
 // (ExecutorAddedEvent.host), since raw executor records: not just findings
 //: reach data.js.
-export function redactExportData(input               )                {
-  const data = redactFailureGroups(input);
+                                                                                       
+
+function redactRunTree                   (input   )    {
+  const data = redactFailureText(input);
   const appIds = new Set        ();
   const hosts = new Set        ();
   const appId = data.app?.id;
@@ -249,5 +262,51 @@ export function redactExportData(input               )                {
   collectHostFields(data.configFindings, hosts);
   collectConfigHostValues(data.app?.config, hosts);
   scanTokens(data, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
-  return applyReplacements(data, { appIds, hosts });
+  const out = applyReplacements(data, { appIds, hosts });
+  if (!out.app) return out;
+  const app = withRedactedName(out.app);
+  // spark.app.name repeats the name in the config the HTML export ships.
+  if (app.config?.['spark.app.name'] == null) return { ...out, app };
+  return { ...out, app: { ...app, config: { ...app.config, 'spark.app.name': app.id ?? '' } } };
+}
+
+/** A run's model and findings with every identifier pseudonymized. Redact this before anything derives
+ * text from the run: the verdict truncates Spark's failure reason, and an
+ * identifier cut by that truncation is a fragment no later pass can match. */
+export function redactRunModel(
+  appModel          ,
+  catalog           ,
+  configFindings           ,
+)                                                                        {
+  // Maps and Sets would lose their entries in the deep copy, so they cross as
+  // tagged plain objects, the way the export payload carries them.
+  const tree = encodeCollections({
+    app: appModel.app,
+    stages: [...appModel.stages.values()],
+    jobs: [...appModel.jobs.values()],
+    sql: [...appModel.sql.values()],
+    executors: appModel.executors,
+    runAggregates: appModel.runAggregates,
+    evidenceAvailability: appModel.evidenceAvailability,
+    catalog,
+    configFindings,
+  })                                                                                                                           ;
+  const redacted = decodeCollections(redactRunTree(tree))               ;
+  const byId =                           (items           ) => new Map((items       ).map((item) => [item.id, item]));
+  return {
+    appModel: {
+      app: redacted.app,
+      stages: byId(redacted.stages),
+      jobs: byId(redacted.jobs),
+      sql: byId(redacted.sql),
+      executors: redacted.executors,
+      runAggregates: redacted.runAggregates                             ,
+      evidenceAvailability: redacted.evidenceAvailability                                    ,
+      // Counts and execution ids, nothing to pseudonymize.
+      skippedLines: appModel.skippedLines,
+      unreadableSqlExecutions: appModel.unreadableSqlExecutions,
+    }            ,
+    catalog: redacted.catalog,
+    configFindings: redacted.configFindings,
+  };
 }

package/vendor-core/remediation.js

@@ -0,0 +1,20 @@
+// Constructors for the structured `remediation` a detector attaches next to its prose
+// `recommendation`: only where the detector already names the Spark property, never an invented one.
+                                                      
+
+                                          
+
+/** The property should be raised; `suggested` is the value when the detector computed one. */
+export function increaseConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'increase', suggested };
+}
+
+/** The property should be lowered; `suggested` is the value when the detector computed one. */
+export function decreaseConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'decrease', suggested };
+}
+
+/** The property should take a specific value (a switch or a class name), not move up or down. */
+export function setConf(key        , suggested            = null)              {
+  return { kind: 'conf', key, direction: 'set', suggested };
+}

package/vendor-core/run-comparison.js

@@ -1,9 +1,16 @@
 import { computeWallClock } from './wall-clock.js';
 import { normalizeDetail } from './detectors.js';
 import { cyrb53 } from './string-hash.js';
+import { computeAllocation } from './allocation.js';
+import { totalExecutorCpuMs, withEarlierAttempts } from './run-totals.js';
+import { planNodesOfStage } from './stage-plan-nodes.js';
 import { captureSnapshot } from './session-snapshot.js';
-                                                                                   
+import { tunedRunNote } from './threshold-overrides.js';
+                                                                                                    
                                                              
+                                                                                        
+import { isIncompleteRun } from './check-coverage.js';
+import { summarizeRunOutcome } from './run-outcome.js';
 
                                  
                              
@@ -12,7 +19,7 @@ import { captureSnapshot } from './session-snapshot.js';
                              
  
                                    
-                                                                                                          
+                                                                                                                        
  
                                     
                                                 
@@ -23,7 +30,16 @@ import { captureSnapshot } from './session-snapshot.js';
                                                                                                                                                   
                                                                             
                                                                             
+                                                                                            
+                                                                                               
+                                   
+                                                                             
  
+
+function jobOutcome(snapshot                 )                    {
+  const { failedJobs, totalJobs } = summarizeRunOutcome(snapshot.jobs, snapshot.catalog);
+  return { failedJobs, totalJobs, incomplete: isIncompleteRun(snapshot.catalog) };
+}
 
 // Replace run-varying tokens (digit runs, long hex ids) with a stable marker so
 // the same logical stage across two runs normalizes to one identity.
@@ -63,23 +79,18 @@ function planTreeIdentity(root                             )                {
 // Falls back to the coarser whole-tree identity when the stage has no
 // attributed nodes (hand-built snapshots without `stageIds`, or unmatched
 // accumulables).
-function sqlNodeIdentity(stage       , snapshot                 )         {
+function sqlNodeIdentity(stage       , snapshot                              )         {
   const execId = stage.sqlExecutionId;
   if (execId == null) return '';
   const root = snapshot.sql.get(execId)?.planTree ?? null;
   if (!root) return '';
-  const fingerprints           = [];
-  (function collect(node          )       {
-    if (node.stageIds?.includes(stage.id)) {
-      fingerprints.push(JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
-    }
-    for (const child of node.children ?? []) collect(child);
-  })(root);
+  const fingerprints = planNodesOfStage(stage, snapshot.sql)
+    .map((node) => JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
   if (fingerprints.length === 0) return planTreeIdentity(root) ?? '';
   return cyrb53(JSON.stringify(fingerprints.sort()));
 }
 
-export function stageIdentity(stage       , snapshot                 )         {
+export function stageIdentity(stage       , snapshot                              )         {
   return normalizeStageName(stage.name ?? '') + '§' + sqlNodeIdentity(stage, snapshot);
 }
 
@@ -186,6 +197,7 @@ function skewRatios(stages         )           {
 export const COMPARISON_METRIC_KEYS                    = [
   'wallClock', 'shuffleSpill', 'taskSkew', 'failedTaskRate', 'diskSpill', 'gcTime',
   'inputBytes', 'outputBytes', 'executorRunTime', 'taskCount', 'executorsAdded',
+  'executorCpuTime', 'allocatedCoreHours',
 ];
 
 // Volume/count metrics, not cost metrics: more or less input/output data, or
@@ -215,20 +227,24 @@ function metric(
 
 export function metricDeltas(baseSnap                 , candSnap                 )                   {
   const out                   = [];
+  // Run totals count every task attempt, failed and speculative ones too (run-totals.ts), the
+  // same sums the CLI metrics block reports; skew stays on each stage's latest attempt.
+  const attemptsOf = (snap                 )          => withEarlierAttempts([...snap.stages.values()]);
 
   // Wall-clock: always computable (computeWallClock tolerates a null app).
   out.push(metric('wallClock', 'Wall-clock duration',
     computeWallClock(baseSnap.app, baseSnap.stages).total,
     computeWallClock(candSnap.app, candSnap.stages).total));
 
-  // Shuffle spill over the whole run: a sum needs no stage matching, and
-  // matching is unreliable on real logs (see plan rationale), so scope it to
-  // all stages exactly like task-skew and failed-rate below.
-  const bSpill = sumField([...baseSnap.stages.values()], 'memoryBytesSpilled');
-  const cSpill = sumField([...candSnap.stages.values()], 'memoryBytesSpilled');
-  out.push(metric('shuffleSpill', 'Shuffle spill',
+  // Memory spill (Spark's memoryBytesSpilled) over the whole run: a sum needs no
+  // stage matching, and matching is unreliable on real logs, so scope it to all
+  // stages exactly like task-skew and failed-rate below. The key stays
+  // `shuffleSpill` so existing --regression-metric callers keep working.
+  const bSpill = sumField(attemptsOf(baseSnap), 'memoryBytesSpilled');
+  const cSpill = sumField(attemptsOf(candSnap), 'memoryBytesSpilled');
+  out.push(metric('shuffleSpill', 'Memory spill',
     bSpill.present ? bSpill.sum : null, cSpill.present ? cSpill.sum : null,
-    { unavailableReason: bSpill.present && cSpill.present ? undefined : 'No shuffle-spill data recorded for a run' }));
+    { unavailableReason: bSpill.present && cSpill.present ? undefined : 'No memory-spill data recorded for a run' }));
 
   // Task skew: p95 of per-stage ratio across the whole run.
   const bSkew = p95(skewRatios([...baseSnap.stages.values()]));
@@ -237,10 +253,10 @@ export function metricDeltas(baseSnap                 , candSnap
     { unavailableReason: bSkew != null && cSkew != null ? undefined : 'No stage had measurable duration for a run' }));
 
   // Failed-task rate: Σ failedTasks / Σ taskCount.
-  const bTasks = sumField([...baseSnap.stages.values()], 'taskCount');
-  const cTasks = sumField([...candSnap.stages.values()], 'taskCount');
-  const bFailed = sumField([...baseSnap.stages.values()], 'failedTasks');
-  const cFailed = sumField([...candSnap.stages.values()], 'failedTasks');
+  const bTasks = sumField(attemptsOf(baseSnap), 'taskCount');
+  const cTasks = sumField(attemptsOf(candSnap), 'taskCount');
+  const bFailed = sumField(attemptsOf(baseSnap), 'failedTasks');
+  const cFailed = sumField(attemptsOf(candSnap), 'failedTasks');
   // Guard on BOTH inputs: a missing `failedTasks` field must render Unavailable,
   // not a false 0% rate (dividing an absent-and-therefore-0 numerator).
   const bRate = bTasks.present && bTasks.sum > 0 && bFailed.present ? bFailed.sum / bTasks.sum : null;
@@ -252,8 +268,8 @@ export function metricDeltas(baseSnap                 , candSnap
   // carries (set in finalizeStage). Correct at any match coverage, like the
   // sums above; no parser or detector change.
   const sumMetric = (key        , label        , field                   , reason        ) => {
-    const b = sumField([...baseSnap.stages.values()], field);
-    const c = sumField([...candSnap.stages.values()], field);
+    const b = sumField(attemptsOf(baseSnap), field);
+    const c = sumField(attemptsOf(candSnap), field);
     out.push(metric(key, label, b.present ? b.sum : null, c.present ? c.sum : null,
       { unavailableReason: b.present && c.present ? undefined : reason }));
   };
@@ -272,6 +288,18 @@ export function metricDeltas(baseSnap                 , candSnap
   out.push(metric('executorsAdded', 'Executors added', bExec, cExec,
     { unavailableReason: bExec != null && cExec != null ? undefined : 'No executor events recorded for a run' }));
 
+  // Executor CPU time (ms) and allocated core-hours cost resources, so less is better. CPU time is
+  // null, not 0, on a run whose log never recorded it (older Spark).
+  const cpuMs = (snap                 )                => totalExecutorCpuMs(attemptsOf(snap));
+  const bCpu = cpuMs(baseSnap), cCpu = cpuMs(candSnap);
+  out.push(metric('executorCpuTime', 'Executor CPU time', bCpu, cCpu,
+    { unavailableReason: bCpu != null && cCpu != null ? undefined : 'No executor CPU time recorded for a run' }));
+  const coreHours = (snap                 )                =>
+    (snap.executors ? computeAllocation(snap).coreHours : null);
+  const bCore = coreHours(baseSnap), cCore = coreHours(candSnap);
+  out.push(metric('allocatedCoreHours', 'Allocated core-hours', bCore, cCore,
+    { unavailableReason: bCore != null && cCore != null ? undefined : 'No executor lifecycle or core count recorded for a run' }));
+
   return out;
 }
 
@@ -282,14 +310,14 @@ export function metricDeltas(baseSnap                 , candSnap
 export function findingsDelta(baseSnap                 , candSnap                 )   
                                                                
   {
-                                                                                                
+                                                                                                              
   const tally = (snap                 )                          => {
     const m = new Map                    (); // `${rule}§${impactBand}` -> { rule, impactBand, count, stages:Set }
     for (const f of snap.catalog) {
-      const rule = typeof f.rule === 'string' ? f.rule : f.type;
+      const rule = 'rule' in f && typeof f.rule === 'string' ? f.rule : f.type;
       const impactBand = f.impactBand ?? 'unknown';
       const key = `${rule}§${impactBand}`;
-      const e = m.get(key) ?? { rule, impactBand, count: 0, stages: new Set        () };
+      const e = m.get(key) ?? { rule, type: f.type, impactBand, count: 0, stages: new Set        () };
       e.count++;
       // Resolve stageId → name on this snapshot only: a single-side lookup, so
       // it needs no cross-run identity. App-level findings (stageId null) add none.
@@ -308,7 +336,7 @@ export function findingsDelta(baseSnap                 , candSnap
     if (candCount === baseCount) continue;
     const meta = ce ?? be ;
     const more = candCount > baseCount ? ce  : be ; // the side with more supplies the labels
-    const row                   = { rule: meta.rule, impactBand: meta.impactBand, baseCount, candCount,
+    const row                   = { rule: meta.rule, type: meta.type, impactBand: meta.impactBand, baseCount, candCount,
       delta: candCount - baseCount, stages: [...more.stages].sort() };
     (candCount > baseCount ? introduced : resolved).push(row);
   }
@@ -428,6 +456,7 @@ export function compareRuns(
     stageSkew: stageSkewDeltas(baseSnap, candSnap, match),
     baseStages: stageList(baseSnap),
     candStages: stageList(candSnap),
+    jobOutcomes: { baseline: jobOutcome(baseSnap), candidate: jobOutcome(candSnap) },
   };
 }
 
@@ -445,8 +474,23 @@ function renderFindingsSection(title        , findings                    )
 // evidence-report.ts's renderMarkdown house style (## section heading, ###
 // subheadings, `- key: value` bullets). Shared by the CLI's --baseline
 // markdown output and the MCP server's compare_runs `format: 'md'`.
-export function renderComparisonMarkdown(comparison                   )         {
+/** `tuned`: tunedDetectors() for the overrides both runs were analyzed with, named in the output
+ * as the evidence report names them. */
+export function renderComparisonMarkdown(
+  comparison                   , verdict                        , tuned                                         ,
+)         {
   const lines = ['', '## Comparison to baseline', ''];
+  if (tuned) lines.push(`- Tuned thresholds (both runs): ${tunedRunNote(tuned)}`, '');
+  if (verdict) {
+    // Names each run by its label (MCP's run IDs), unless the labels are just the role names the
+    // CLI passes, where the verdict below already says baseline and candidate.
+    if (comparison.baselineLabel !== 'baseline' || comparison.candidateLabel !== 'candidate') {
+      lines.push(`Baseline: ${comparison.baselineLabel} · Candidate: ${comparison.candidateLabel}`, '');
+    }
+    lines.push(verdict.title);
+    if (verdict.sentences.length > 0) lines.push('', verdict.sentences.join(' '));
+    lines.push('');
+  }
   if (comparison.confidence === 'low') {
     lines.push(`- confidence: low, ${comparison.reason}`);
     lines.push('');