sparkforensics-server 0.5.1 → 0.6.0

package/vendor-core/impact-model.js

@@ -118,8 +118,8 @@ export function tasksMostlyIdle(stage       , sql                            = n
   return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
 }
 
-// skew, straggler and stageSlowness claim time off the stage's longest task itself, so the
-// occupancy clip must not floor them at that same task (see estimateSingleStage).
+// skew, straggler, stageSlowness and lowShuffleParallelism claim time off the stage's longest task
+// itself, so the occupancy clip must not floor them at that same task (see estimateSingleStage).
 export const TAIL_CLAIM                             = { shortensLongestTask: true };
 
 // No quantifiable magnitude -> 'informational'; a rawWaste figure with no stage window ->

package/vendor-core/mcp-server-factory.js

@@ -119,14 +119,18 @@ export function createMcpServer({ thresholds }
       maxRegressionPct: z.number().optional(), regressionMetric: z.string().optional(), failOnIntroduced: z.string().optional(),
       regressionBudgets: z.array(z.object({ metric: z.string(), maxPct: z.number().nonnegative() })).optional()
         .describe('Further regression budgets, one per metric (the CLI\'s repeated --regression-budget). A metric can be budgeted once across this list and maxRegressionPct/regressionMetric.'),
+      stageRegressionBudgets: z.array(z.object({ metric: z.string(), maxPct: z.number().nonnegative() })).optional()
+        .describe('Per paired stage (needs sourceB/runIdB): fail when any eligible pair\'s metric (executorRunTime, executorCpuTime, memoryBytesSpilled, diskBytesSpilled, shuffleReadBytes, shuffleWriteBytes) regressed by more than maxPct. One entry per metric. Inconclusive when no stage pairs.'),
+      stageQualities: z.array(z.enum(['exact', 'structural', 'aligned'])).optional()
+        .describe('Pair qualities the stage budgets read. Default exact and structural: an aligned pair may compare different work.'),
     },
   }, ({
     source, runId, runIdB, sourceB,
     maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct,
-    maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets,
+    maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets, stageRegressionBudgets, stageQualities,
   }) => toolResult(evaluateBudgetsForRun(
     { source, runId },
-    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets },
+    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced, regressionBudgets, stageRegressionBudgets, stageQualities },
     (runIdB || sourceB) ? { runId: runIdB, source: sourceB } : undefined,
     { thresholds },
   )));

package/vendor-core/mcp-tools.js

@@ -13,7 +13,7 @@ import { buildComparisonOutput,                       } from './comparison-outpu
 import { compileNormalizePatterns } from './stage-alignment.js';
 import { runOutputBlocks } from './run-output.js';
 import { evaluateBudgets,                                       } from './cli/budgets.js';
-import { combineRegressionBudgets } from './cli/regression-budgets.js';
+import { assertStageBudgetMetric, combineRegressionBudgets } from './cli/regression-budgets.js';
 import { FINDING_NAMES, titleCase } from './finding-names.js';
 import { docAnchorForType } from './detector-docs.js';
 import { DETECTORS,                         } from './detectors.js';
@@ -416,6 +416,15 @@ export async function evaluateBudgetsForRun(
   } catch (e) {
     throw mcpError('access-or-upstream-failure', (e         ).message);
   }
+  if (budgets.stageQualities !== undefined && budgets.stageRegressionBudgets === undefined) {
+    throw mcpError('access-or-upstream-failure', 'stageQualities requires stageRegressionBudgets.');
+  }
+  try {
+    for (const { metric } of budgets.stageRegressionBudgets ?? []) assertStageBudgetMetric(metric, 'stageRegressionBudgets');
+    combineRegressionBudgets((budgets.stageRegressionBudgets ?? []).map((budget) => ({ origin: 'stageRegressionBudgets', budget })));
+  } catch (e) {
+    throw mcpError('access-or-upstream-failure', (e         ).message);
+  }
   const [first, second] = await Promise.all([
     resolveAndAnalyze(primary, opts?.thresholds),
     secondary ? resolveAndAnalyze(secondary, opts?.thresholds) : Promise.resolve(undefined),

package/vendor-core/median.js

@@ -0,0 +1,8 @@
+// The textbook median of values already sorted ascending: the middle value for an odd count, the
+// mean of the two middle values for an even one. 0 for an empty list.
+export function medianOfSorted(sorted                   )         {
+  const n = sorted.length;
+  if (n === 0) return 0;
+  const mid = n >> 1;
+  return n % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}

package/vendor-core/remediation.js

@@ -1,20 +1,25 @@
 // Constructors for the structured `remediation` a detector attaches next to its prose
 // `recommendation`: only where the detector already names the Spark property, never an invented one.
-                                                      
+                                                                           
 
-                                          
+                                              
 
 /** The property should be raised; `suggested` is the value when the detector computed one. */
-export function increaseConf(key        , suggested            = null)              {
+export function increaseConf(key        , suggested            = null)                  {
   return { kind: 'conf', key, direction: 'increase', suggested };
 }
 
 /** The property should be lowered; `suggested` is the value when the detector computed one. */
-export function decreaseConf(key        , suggested            = null)              {
+export function decreaseConf(key        , suggested            = null)                  {
   return { kind: 'conf', key, direction: 'decrease', suggested };
 }
 
 /** The property should take a specific value (a switch or a class name), not move up or down. */
-export function setConf(key        , suggested            = null)              {
+export function setConf(key        , suggested            = null)                  {
   return { kind: 'conf', key, direction: 'set', suggested };
 }
+
+/** No property fixes it: the job's code or data has to change, as `hint` says. */
+export function codeFix(hint        )                  {
+  return { kind: 'code', hint };
+}

package/vendor-core/run-comparison.js

@@ -38,6 +38,7 @@ import { summarizeRunOutcome } from './run-outcome.js';
                                          
                                          
                                                              
+                                                           
                             
                                                                              
                             
@@ -408,6 +409,7 @@ export function compareRuns(
     unmatched: alignment.unmatched,
     replanned: alignment.replanned,
     bookkeepingStageIds: alignment.bookkeepingStageIds,
+    executionAlignment: alignment.executionAlignment,
     metrics: metricDeltas(baseSnap, candSnap),
     findings: findingsDelta(baseSnap, candSnap),
     stageSkew: stageSkewDeltas(baseSnap, candSnap, alignment.pairs),

package/vendor-core/run-output.js

@@ -5,21 +5,32 @@ import { buildEffectiveConf,                    } from './effective-conf.js';
                                                          
                                            
 
+/** Which build wrote a report: the host package's name and version, and the build id of the core it
+ * loaded. Core does not know its host, so the surface passes it in. */
+                                  
+               
+                  
+                  
+ 
+
 /** The CLI report's `metrics` and `effectiveConf` blocks, shared with MCP diagnose_run. Under
  * `redact` they come from the run redacted with the report's own inputs, so host and app
- * pseudonyms match the report's. `confKeys`/`confRedactRegex` narrow and mask `effectiveConf`. */
+ * pseudonyms match the report's. `confKeys`/`confRedactRegex` narrow and mask `effectiveConf`.
+ * `generator`, when the surface supplies it, is returned as given: it holds no run identifiers. */
 export function runOutputBlocks(
   appModel          ,
-  { redact, thresholds, confKeys, confRedactRegex }   
+  { redact, thresholds, confKeys, confRedactRegex, generator }   
                                                                                                      
+                                
     = {},
-)                                                               {
+)                                                                                            {
   let blocksModel = appModel;
   if (redact) {
     const { catalog, config } = runFindings(appModel, thresholds);
     blocksModel = redactRunModel(appModel, catalog, config).appModel;
   }
   return {
+    ...(generator ? { generator } : {}),
     metrics: computeRunMetrics(blocksModel, thresholds),
     effectiveConf: buildEffectiveConf(blocksModel.app, { keys: confKeys, userPattern: confRedactRegex }),
   };

package/vendor-core/sequence-alignment.js

@@ -0,0 +1,83 @@
+// Order-preserving sequence alignment: the weighted longest-common-subsequence dynamic program the
+// stage aligner runs over each run's SQL executions. Pure and deterministic.
+//
+// Cost: the full table has n*m cells and one similarity call per cell. Above `FULL_ALIGNMENT_MAX_CELLS`
+// the table is restricted to a band around the diagonal, which costs about n*(2W+1) cells with
+// W = `BAND_BASE_HALF_WIDTH` + |n-m|/2, and the result reports that the bounded form ran.
+
+/** Largest n*m the alignment runs in full (1000 x 1000 executions). */
+export const FULL_ALIGNMENT_MAX_CELLS = 1_000_000;
+/** Band half-width added to half of |n-m| once the table is bounded: the band is centered on the line
+ * from corner to corner, and a run of insertions or deletions in one place drifts the best path from
+ * that line by at most |n-m|/2. */
+export const BAND_BASE_HALF_WIDTH = 100;
+
+                                    
+                                                        
+                                 
+                                                                          
+                   
+ 
+
+/** Aligns two sequences of lengths `n` and `m`, maximizing the summed `score` of matched pairs. `score`
+ * returns a non-negative integer; a pair scoring below `minScore` is never matched. Of equally good
+ * alignments the one that matches whenever matching is optimal wins, and where it must skip, it skips the
+ * element with the higher id (`idA`, `idB`), so the lower ids stay and swapping the two sequences mirrors
+ * the pairs. */
+export function alignSequences(
+  n        , m        ,
+  score                                  ,
+  minScore        ,
+  idA                       , idB                       ,
+)                    {
+  if (n === 0 || m === 0) return { pairs: [], bounded: false };
+  const bounded = n * m > FULL_ALIGNMENT_MAX_CELLS;
+  const half = BAND_BASE_HALF_WIDTH + Math.ceil(Math.abs(n - m) / 2);
+  // Row i may match columns lo[i]..hi[i]; both bounds only grow with i.
+  const lo = new Int32Array(n), hi = new Int32Array(n);
+  for (let i = 0; i < n; i++) {
+    const center = Math.floor((i * m) / n);
+    lo[i] = bounded ? Math.max(0, center - half) : 0;
+    hi[i] = bounded ? Math.min(m - 1, center + half) : m - 1;
+  }
+  // best[i][j - lo[i]]: the best total from the suffixes starting at (i, j), for j inside row i's band.
+  const best               = new Array(n);
+  // Cells outside the band are not stored: left of it the row's first cell stands in (those columns can
+  // only match in later rows, whose bands start no earlier), right of it the row cannot match, so the
+  // next row answers.
+  const get = (i        , j        )         => {
+    for (;;) {
+      if (i >= n || j >= m) return 0;
+      if (j < lo[i]) j = lo[i];
+      else if (j > hi[i]) i++;
+      else return best[i][j - lo[i]];
+    }
+  };
+  // The caller memoizes `score`, so the walk below re-asking for visited cells is cheap.
+  const cell = score;
+  for (let i = n - 1; i >= 0; i--) {
+    const row = new Int32Array(hi[i] - lo[i] + 1);
+    best[i] = row;
+    for (let j = hi[i]; j >= lo[i]; j--) {
+      let v = Math.max(get(i + 1, j), j + 1 <= hi[i] ? row[j + 1 - lo[i]] : get(i + 1, j + 1));
+      const s = cell(i, j);
+      if (s >= minScore) v = Math.max(v, s + get(i + 1, j + 1));
+      row[j - lo[i]] = v;
+    }
+  }
+
+  const pairs                          = [];
+  let i = 0, j = 0;
+  while (i < n && j < m) {
+    if (j < lo[i]) { j = lo[i]; continue; }
+    if (j > hi[i]) { i++; continue; }
+    const v = best[i][j - lo[i]];
+    const s = cell(i, j);
+    if (s >= minScore && s + get(i + 1, j + 1) === v) { pairs.push([i, j]); i++; j++; continue; }
+    const skipA = get(i + 1, j) === v, skipB = get(i, j + 1) === v;
+    if (skipA && skipB) { if (idA(i) >= idB(j)) i++; else j++; }
+    else if (skipA) i++;
+    else j++;
+  }
+  return { pairs, bounded };
+}