sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/cli/collect-run.js

@@ -1,4 +1,4 @@
-import { readFileSync, readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
+import { readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
 import { join, basename } from 'node:path';
 import { createState, runParse, runParseFiles, reassembleRollingEntries } from '../parser-worker.js';
 import { nodeParseCodecs } from './native-zstd.js';
@@ -6,10 +6,13 @@ import { createModelCallbacks } from '../model-assembler.js';
 import { routeMessage,                     } from '../ingest.js';
                                             
 
+// No whole-file arrayBuffer(): the parser only ever reads bounded slices, and a
+// whole-file read is what capped local logs at 2 GiB.
                            
                              
                                                                              
-                                      
+                                                                                   
+                
  
 
 export function emptyAppModel()           {
@@ -24,18 +27,48 @@ export function emptyAppModel()           {
   };
 }
 
-// File-like shape runParse/runParseFiles need: name, size, slice().arrayBuffer(), arrayBuffer().
+// File-like shape runParse/runParseFiles need (name, size, slice().arrayBuffer()), read with
+// positioned readSync so only the requested slice is ever in memory, whatever the file size.
+// The descriptor opens on the first read and closes once a read reaches the end of the file,
+// so a rolling directory's parts hold at most one open descriptor at a time while streaming.
+// A later read (a zip archive reads its tail first) reopens it; callers must still call
+// close() once parsing settles, to cover reads that stopped early on an error.
 export function nodeFileFromPath(path        )           {
-  const bytes = readFileSync(path);
-  const u8 = new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const name = basename(path);
+  const size = statSync(path).size;
+  let fd                = null;
+  const close = () => {
+    if (fd === null) return;
+    const open = fd;
+    fd = null;
+    closeSync(open);
+  };
   return {
-    name: basename(path),
-    size: u8.length,
+    name,
+    size,
     slice(start        , end        ) {
-      const view = u8.subarray(start, end);
-      return { async arrayBuffer() { return view.slice().buffer; } };
+      return {
+        async arrayBuffer()                       {
+          const from = Math.max(0, start);
+          const to = Math.min(end, size);
+          if (to <= from) return new ArrayBuffer(0);
+          const buf = new Uint8Array(to - from);
+          fd ??= openSync(path, 'r');
+          // readSync may return fewer bytes than asked; loop until the slice is full.
+          for (let filled = 0; filled < buf.length;) {
+            const n = readSync(fd, buf, filled, buf.length - filled, from + filled);
+            if (n === 0) {
+              close();
+              throw new Error(`"${name}" shrank while it was being read`);
+            }
+            filled += n;
+          }
+          if (to === size) close();
+          return buf.buffer;
+        },
+      };
     },
-    async arrayBuffer() { return u8.slice().buffer; },
+    close,
   };
 }
 
@@ -107,26 +140,38 @@ export function collectViaDispatch(
 
 export async function collectRun(inputPath        )                                                        {
   const stat = statSync(inputPath);
-  return collectViaDispatch((state, emit, reject) => {
-    if (stat.isDirectory()) {
-      if (!isRollingLogDirectory(inputPath)) {
-        reject(new Error("This isn't a Spark rolling event-log directory. Pass a single event-log file instead."));
-        return;
-      }
-      const names = readdirSync(inputPath);
-      let ordered;
-      try {
-        ordered = reassembleRollingEntries(names);
-      } catch (e) {
-        reject(e);
-        return;
+  // Every file opened for this run, closed once parsing settles on any path (done, parse
+  // error, decode error, or a throw past the parser's guards).
+  const opened             = [];
+  const open = (path        ) => {
+    const file = nodeFileFromPath(path);
+    opened.push(file);
+    return file;
+  };
+  try {
+    return await collectViaDispatch((state, emit, reject) => {
+      if (stat.isDirectory()) {
+        if (!isRollingLogDirectory(inputPath)) {
+          reject(new Error("This isn't a Spark rolling event-log directory. Pass a single event-log file instead."));
+          return;
+        }
+        const names = readdirSync(inputPath);
+        let ordered;
+        try {
+          ordered = reassembleRollingEntries(names);
+        } catch (e) {
+          reject(e);
+          return;
+        }
+        const files = ordered.map((name) => open(join(inputPath, name)));
+        // .catch(reject), not void: a throw past the parser's guards would otherwise leave
+        // this Promise pending forever, surfacing only as an unhandled rejection.
+        runParseFiles(files, state, { emit, ...nodeParseCodecs }).catch(reject);
+      } else {
+        runParse(open(inputPath), state, { emit, ...nodeParseCodecs }).catch(reject);
       }
-      const files = ordered.map((name) => nodeFileFromPath(join(inputPath, name)));
-      // .catch(reject), not void: a throw past the parser's guards would otherwise leave
-      // this Promise pending forever, surfacing only as an unhandled rejection.
-      runParseFiles(files, state, { emit, ...nodeParseCodecs }).catch(reject);
-    } else {
-      runParse(nodeFileFromPath(inputPath), state, { emit, ...nodeParseCodecs }).catch(reject);
-    }
-  }, (msg) => new Error((msg                       ).message));
+    }, (msg) => new Error((msg                       ).message));
+  } finally {
+    for (const file of opened) file.close();
+  }
 }

package/vendor-core/cli/native-zstd.js

@@ -343,8 +343,8 @@ export function createThreadedZstdDecoder(
 }
 
 // Node's native zstd where this Node has it (22.15+/23.8+); older Nodes keep the vendored fzstd.
-// runParse/runParseFiles (collectRun) take the threaded decoder. decodeShsArchive decodes each
-// archive entry in one synchronous call, so the SHS archive loader takes the inline one.
+// runParse/runParseFiles (collectRun) take the threaded decoder; the SHS archive loader
+// (decodeShsArchive over an in-memory download) takes the inline one.
 export const nodeParseCodecs                                                     =
   nativeZstdAvailable ? { zstdDecoder: createThreadedZstdDecoder } : {};
 export const nodeArchiveCodecs                                                   =

package/vendor-core/cli/threshold-config.js

@@ -0,0 +1,28 @@
+// Reads a --thresholds config file for the CLI and the MCP server. Every failure throws an Error
+// naming the file and the problem, so the caller refuses to run instead of quietly falling back
+// to the defaults the user meant to change.
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { parseThresholdOverrides } from '../threshold-overrides.js';
+                                                          
+
+export function loadThresholdOverrides(path        )                     {
+  const fullPath = resolve(path);
+  let text        ;
+  try {
+    text = readFileSync(fullPath, 'utf8');
+  } catch (e) {
+    throw new Error(`Cannot read thresholds file ${fullPath}: ${(e         ).message}`, { cause: e });
+  }
+  let raw         ;
+  try {
+    raw = JSON.parse(text);
+  } catch (e) {
+    throw new Error(`Thresholds file ${fullPath} is not valid JSON: ${(e         ).message}`, { cause: e });
+  }
+  try {
+    return parseThresholdOverrides(raw);
+  } catch (e) {
+    throw new Error(`Thresholds file ${fullPath}: ${(e         ).message}`, { cause: e });
+  }
+}

package/vendor-core/comparison-verdict.js

@@ -0,0 +1,177 @@
+import { formatDuration, typeTag } from './format-utils.js';
+import { TAG_HELP } from './finding-tag-help.js';
+import { NEUTRAL_METRIC_KEYS,                        } from './run-comparison.js';
+
+/** The slice of a comparison metric row the verdict reads. */
+                                
+              
+                
+                          
+                           
+                       
+                                                                                    
+ 
+
+/** The slice of a finding-category delta the verdict reads. */
+                                  
+               
+                    
+                    
+ 
+
+/** How many of a run's ended jobs failed, counted as `summarizeRunOutcome`
+ * counts them. */
+                                    
+                     
+                    
+                                                                            
+                               
+                       
+ 
+
+                                                                     
+
+                                        
+                
+                       
+                      
+ 
+
+/** A change under this share of run A's value reads as "about the same", for
+ * run time and cost metrics alike: run-to-run noise on a shared cluster easily
+ * moves a job a percent or two. */
+export const SAME_CHANGE_SHARE = 0.02;
+
+function measured(metric               )                                                                    {
+  return metric.baseline != null && metric.candidate != null;
+}
+
+function movedPastNoise(metric                                                         )          {
+  if (metric.baseline === 0) return metric.candidate !== 0;
+  return Math.abs(metric.candidate - metric.baseline) / Math.abs(metric.baseline) >= SAME_CHANGE_SHARE;
+}
+
+/** Plain name for a finding type ("Memory and disk spill"), falling back to
+ * its tag when the tag has no help entry. */
+function categoryName(type        )         {
+  const tag = typeTag(type);
+  return TAG_HELP[tag]?.expansion ?? tag;
+}
+
+/** Net count change per displayed category name. Categories are tallied per
+ * (rule, impact band), and several rules share one name (every Plan Advisor
+ * type reads "Plan advisor", every stage-shape sub-rule "Stage shape"), so a
+ * rule whose findings moved from critical to warning, or two rules under one
+ * name moving opposite ways, would otherwise
+ * read as both "introduced" and "resolved". Order follows first appearance,
+ * introduced before resolved. */
+function netByCategory(findings                                                                )                      {
+  const net = new Map                ();
+  for (const item of [...findings.introduced, ...findings.resolved]) {
+    const name = categoryName(item.type);
+    net.set(name, (net.get(name) ?? 0) + item.candCount - item.baseCount);
+  }
+  return net;
+}
+
+function namesWhere(net                     , keep                             )           {
+  return [...net].filter(([, change]) => keep(change)).map(([name]) => name);
+}
+
+/** "Run B had 2 of 5 jobs fail", or, when every job failed, "Run B's only
+ * job failed" / "All 3 of run B's jobs failed" rather than "1 of 1 jobs". */
+function jobsFailed(run           , { failedJobs, totalJobs }                   )         {
+  if (failedJobs < totalJobs) return `Run ${run} had ${failedJobs} of ${totalJobs} jobs fail`;
+  return totalJobs === 1 ? `Run ${run}'s only job failed` : `All ${totalJobs} of run ${run}'s jobs failed`;
+}
+
+/** Run A's failures in the parenthesis after run B's. */
+function baselineFailures({ failedJobs, totalJobs }                   )         {
+  if (failedJobs === 0) return 'none';
+  if (failedJobs < totalJobs) return `${failedJobs} of ${totalJobs}`;
+  return totalJobs === 1 ? 'its only job failed' : `all ${totalJobs} failed`;
+}
+
+/** The headline when either run had failed jobs, as the run verdict leads
+ * with a failure: a faster run B that dropped work is not an improvement,
+ * and with equal failure counts the tone stays neutral since a failing run
+ * that ends sooner may just have failed earlier. Null when both runs
+ * completed. */
+function failureHeadline(base                   , cand                   )                                                 {
+  if (base.failedJobs === 0 && cand.failedJobs === 0) return null;
+  const tone = cand.failedJobs > base.failedJobs ? 'worse' : cand.failedJobs < base.failedJobs ? 'better' : 'same';
+  if (cand.failedJobs === 0) return { title: `${jobsFailed('A', base)}; run B completed`, tone };
+  return { title: `${jobsFailed('B', cand)} (run A: ${baselineFailures(base)})`, tone };
+}
+
+/** One plain answer to "did run B get better or worse than run A", from the
+ * comparison's own whole-run metrics and finding-category tallies: run time
+ * first, then which cost metrics moved each way past run-to-run noise, then which finding
+ * categories appeared or went away. When either run had failed jobs, that
+ * leads instead and run time becomes the first sentence. When either log is
+ * incomplete, run time is stated as what each log covers, never as faster or
+ * slower, and the tone stays neutral. Volume and count metrics (input, output,
+ * tasks, executors) are left out because more or less of them is not
+ * inherently better or worse. */
+export function summarizeComparison(
+  metrics                 ,
+  findings                                                                ,
+  jobs                                                                ,
+)                        {
+  const wall = metrics.find((metric) => metric.key === 'wallClock');
+  let title = 'Run time could not be compared between run A and run B';
+  let tone                 = 'unknown';
+  // A log with no end-of-run record stops where the run was cut off, so its
+  // shorter time is not a speed-up: say how much each log covers, neutrally.
+  const incompleteRuns = jobs ? (['A', 'B']         ).filter((run) => (run === 'A' ? jobs.baseline : jobs.candidate).incomplete) : [];
+  if (wall && wall.baseline != null && wall.baseline > 0 && wall.candidate != null && incompleteRuns.length > 0) {
+    const change = wall.candidate - wall.baseline;
+    title = Math.abs(change / wall.baseline) < SAME_CHANGE_SHARE
+      ? "Run B's log covers about as much run time as run A's"
+      : `Run B's log covers ${formatDuration(Math.abs(change))} ${change < 0 ? 'less' : 'more'} run time than run A's`;
+  } else if (wall && wall.baseline != null && wall.baseline > 0 && wall.candidate != null) {
+    const change = wall.candidate - wall.baseline;
+    const share = change / wall.baseline;
+    if (Math.abs(share) < SAME_CHANGE_SHARE) {
+      title = 'Run B took about as long as run A';
+      tone = 'same';
+    } else {
+      const faster = change < 0;
+      title = `Run B finished ${formatDuration(Math.abs(change))} ${faster ? 'faster' : 'slower'} than run A (${Math.round(Math.abs(share) * 100)}%)`;
+      tone = faster ? 'better' : 'worse';
+    }
+  }
+
+  const cost = metrics.filter((metric) => metric.key !== 'wallClock' && !NEUTRAL_METRIC_KEYS.has(metric.key)).filter(measured);
+  const moved = cost.filter(movedPastNoise);
+  const worse = moved.filter((metric) => metric.direction === 'regression').map((metric) => metric.label);
+  const better = moved.filter((metric) => metric.direction === 'improvement').map((metric) => metric.label);
+  const sentences           = [];
+  const failure = jobs ? failureHeadline(jobs.baseline, jobs.candidate) : null;
+  if (failure) {
+    sentences.push(`${title}.`);
+    title = failure.title;
+    tone = failure.tone;
+  }
+  if (incompleteRuns.length === 2) {
+    sentences.push('Neither log has an end-of-run record, so their times cover only what each log captured, not how long the runs took.');
+  } else if (incompleteRuns.length === 1) {
+    sentences.push(`Run ${incompleteRuns[0]}'s log has no end-of-run record, so its time covers only what the log captured, not how long the run took.`);
+  }
+  if (worse.length > 0) sentences.push(`Worse in run B: ${worse.join(', ')}.`);
+  if (better.length > 0) sentences.push(`Better in run B: ${better.join(', ')}.`);
+  if (cost.length > 0 && worse.length === 0 && better.length === 0) sentences.push('Other measured cost metrics look about the same.');
+
+  const net = netByCategory(findings);
+  const introduced = namesWhere(net, (change) => change > 0);
+  const resolved = namesWhere(net, (change) => change < 0);
+  if (introduced.length > 0) sentences.push(`New or more frequent in run B: ${introduced.join(', ')}.`);
+  if (resolved.length > 0) sentences.push(`Less frequent in run B: ${resolved.join(', ')}.`);
+  return { title, tone, sentences };
+}
+
+/** The comparison verdict for a core comparison result: the one the dashboard's comparison page
+ * leads with, and the one the CLI's --baseline report and MCP compare_runs carry. */
+export function comparisonVerdict(result                                                                 )                        {
+  return summarizeComparison(result.metrics, result.findings, result.jobOutcomes);
+}

package/vendor-core/core-source-hash.txt

@@ -0,0 +1 @@
+b5b0e0b6178129de4de482d4b59fbf881822655d8911b374c4a60a33b38f63fb

package/vendor-core/core-usage-locality.js

@@ -18,6 +18,8 @@ export const LOCALITY_TIERS           = ['PROCESS_LOCAL', 'NODE_LOCAL', 'RACK_LO
                                     
                    
                                    
+                                                                                                 
+                  
  
 
 export function computeLocalityAreaSeries(
@@ -25,7 +27,7 @@ export function computeLocalityAreaSeries(
   { bucketWidthMs = 60_000, tiers = LOCALITY_TIERS }                            = {},
 )                           {
   const valid = stages.filter(s => (s.completedAt ?? 0) > (s.submittedAt ?? 0) && (s.executorRunTime ?? 0) > 0);
-  if (valid.length === 0) return { labels: [], series: {} };
+  if (valid.length === 0) return { labels: [], series: {}, endTime: 0 };
   const startTime = valid.reduce((m, s) => Math.min(m, s.submittedAt ?? m), Infinity);
   const endTime = valid.reduce((m, s) => Math.max(m, s.completedAt ?? m), -Infinity);
   const nBuckets = Math.max(1, Math.ceil((endTime - startTime) / bucketWidthMs));
@@ -65,5 +67,57 @@ export function computeLocalityAreaSeries(
 
   const labels           = [];
   for (let b = 0; b < nBuckets; b++) labels.push(startTime + b * bucketWidthMs);
-  return { labels, series };
+  return { labels, series, endTime };
 }
+
+/** How many time buckets the Core Usage by Locality chart aims for across a run. */
+export const LOCALITY_CHART_TARGET_BUCKETS = 60;
+
+                                     
+                                
+            
+                         
+ 
+
+                           
+                          
+                                                                                            
+
+/** The Core Usage by Locality chart's points and its "busy at the peak" figure, shared by the
+ * dashboard widget and the CLI/MCP run summary. Buckets are at least a minute wide; a last bucket
+ * the stages only partly cover is rescaled to the covered part, and each bucket's idle cores are
+ * the peak minus its busy cores. */
+export function buildLocalityChart(
+  stages                 ,
+  app                                                               ,
+  targetBuckets         = LOCALITY_CHART_TARGET_BUCKETS,
+)                {
+  const hasActivity = stages.some((s) => (s.executorRunTime ?? 0) > 0 && (s.completedAt ?? 0) > (s.submittedAt ?? 0));
+  if (!hasActivity) return { hasActivity: false };
+
+  const start = app?.startTime ?? 0;
+  const end = app?.endTime ?? start;
+  const bucketWidthMs = Math.max(60_000, Math.ceil(Math.max(1, end - start) / targetBuckets));
+  const { labels, series, endTime: seriesEnd } = computeLocalityAreaSeries(stages, { bucketWidthMs });
+  const order = [...LOCALITY_TIERS.filter((t) => series[t]), ...(series.OTHER ? ['OTHER'] : []), 'idle'];
+
+  const points                       = labels.map((t, i) => {
+    const point                     = { t: Math.round((t - start) / 1000) };
+    // The series averages each bucket over its full width, so a last bucket
+    // that runs past the series' own end (or a run shorter than one bucket)
+    // reads diluted: a 10s stage in a 60s bucket showed well under its busy
+    // cores. Rescale to the part of the bucket the series actually covers.
+    const coveredMs = Math.min(bucketWidthMs, seriesEnd - t);
+    const scale = bucketWidthMs / coveredMs;
+    for (const tier of order) if (tier !== 'idle') point[tier] = (series[tier]?.[i] ?? 0) * scale;
+    return point;
+  });
+  const busyTiers = order.filter((t) => t !== 'idle');
+  const busyTotal = (p                    ) => busyTiers.reduce((sum, t) => sum + p[t], 0);
+  const peakCores = points.reduce((max, p) => Math.max(max, busyTotal(p)), 0);
+  // Same rule as the core series (peak busy minus each bucket's busy), redone
+  // on the rescaled values so the stack still tops out at the peak.
+  for (const p of points) p.idle = Math.max(0, peakCores - busyTotal(p));
+  return { hasActivity: true, order, points, peakCores };
+}
+

package/vendor-core/detector-docs.js

@@ -0,0 +1,58 @@
+// Type-level doc anchors, read off the detector catalog. Kept out of docs-config.ts so the docs
+// URL helpers (used by every renderer) don't pull in the detectors themselves.
+import { DETECTORS, ENTRY_BY_TYPE } from './detectors.js';
+import { isKnownDocAnchor } from './docs-config.js';
+import { getThresholdSummary } from './threshold-summary.js';
+                                               
+
+// DETECTORS is static, so this grouping is built once (lazily) instead of re-scanning per
+// docAnchorForType call (called once per TagBadge per render). Keyed by emitted finding type, so
+// broadcastSizing's anchor lands on underBroadcast/overBroadcast.
+let anchorsByTypeCache                                                  ;
+
+function anchorsByType()                                       {
+  if (!anchorsByTypeCache) {
+    anchorsByTypeCache = new Map();
+    for (const entry of DETECTORS                       ) {
+      for (const type of entry.emits) {
+        const anchors = anchorsByTypeCache.get(type) ?? new Set();
+        anchors.add(entry.docAnchor);
+        anchorsByTypeCache.set(type, anchors);
+      }
+    }
+  }
+  return anchorsByTypeCache;
+}
+
+/** Resolves a finding `type` to its documented anchor from the DETECTORS entries that emit it.
+ * Returns undefined when entries sharing the type disagree on docAnchor (only configAudit today),
+ * or when the resolved anchor isn't in the allowlist (isKnownDocAnchor, the same gate DocsLink uses). */
+export function docAnchorForType(type        )                     {
+  const anchors = anchorsByType().get(type) ?? new Set();
+  if (anchors.size !== 1) return undefined;
+  const [anchor] = anchors;
+  return anchor && isKnownDocAnchor(anchor) ? anchor : undefined;
+}
+
+/** What a renderer shows about one finding type without running its detector. */
+                               
+                                                       
+                
+                                                                              
+                           
+                                                          
+                           
+                                                                                             
+                           
+ 
+
+/** Every emitted finding type's `DetectorInfo`, in `DETECTORS` declaration order of each entry's
+ * `emits` list (a type repeated across entries takes its first position, as `ENTRY_BY_TYPE` does).
+ * The key order is the widget order's tie-break, so it is part of the result. */
+export function detectorInfoByType()                               {
+  const info                               = {};
+  for (const [type, { order, scope }] of ENTRY_BY_TYPE) {
+    info[type] = { order, docAnchor: docAnchorForType(type) ?? null, thresholdSummary: getThresholdSummary(type), scope };
+  }
+  return info;
+}