sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/threshold-overrides.js

@@ -0,0 +1,160 @@
+// User threshold overrides: validation of a parsed config file, and which of a detector's
+// thresholds an override actually moved off its specification default. Only the CLI and the MCP
+// server accept overrides; the dashboard always runs the defaults. Reading the file is
+// cli/threshold-config.ts's job, so this module stays free of Node APIs.
+import {
+  DETECTORS, ENTRY_BY_TYPE, detectorCatalog,                                                                                            
+} from './detectors.js';
+                                                                  
+
+const entries                      = DETECTORS;
+
+function isPlainObject(value         )                                   {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isNonNegativeNumber(value         )                  {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0;
+}
+
+// Entries a user may tune: not config-scope (those audits compare against Spark's own defaults,
+// such as its memoryOverhead floor), and with at least one threshold.
+function tunableEntry(type        )           {
+  const entry = entries.find((d) => d.type === type);
+  if (!entry) {
+    const tunable = entries.filter((d) => d.scope !== 'config' && Object.keys(d.thresholds).length > 0).map((d) => d.type);
+    throw new Error(`unknown detector "${type}" (tunable detectors: ${tunable.join(', ')})`);
+  }
+  if (entry.scope === 'config') throw new Error(`"${type}" is not tunable: its checks compare against Spark's own defaults`);
+  if (Object.keys(entry.thresholds).length === 0) throw new Error(`"${type}" has no thresholds to tune`);
+  return entry;
+}
+
+function checkValue(type        , name        , fallback                            , value         )                             {
+  if (!Array.isArray(fallback)) {
+    if (!isNonNegativeNumber(value)) throw new Error(`"${type}.${name}" must be a non-negative number`);
+    return value;
+  }
+  // Tier tables are indexed by position and read as ascending bands.
+  const ascending = Array.isArray(value) && value.length === fallback.length
+    && value.every(isNonNegativeNumber) && value.every((v, i) => i === 0 || v >= value[i - 1]);
+  if (!ascending) throw new Error(`"${type}.${name}" must be an ascending list of ${fallback.length} non-negative numbers`);
+  return Object.freeze([...value]);
+}
+
+/**
+ * Validates a parsed thresholds config: `{ "<detector type>": { "<threshold>": value } }`, with each
+ * value shaped like that threshold's default (see a report's `detectors` catalog for the names,
+ * units and defaults). Throws an Error naming the first problem; never drops a bad entry silently.
+ */
+export function parseThresholdOverrides(raw         )                     {
+  if (!isPlainObject(raw)) throw new Error('expected a JSON object keyed by detector type');
+  const result                                     = {};
+  for (const [type, perDetector] of Object.entries(raw)) {
+    const entry = tunableEntry(type);
+    if (!isPlainObject(perDetector)) throw new Error(`"${type}" must be an object of threshold values`);
+    const values                                             = {};
+    for (const [name, value] of Object.entries(perDetector)) {
+      // Own keys only: an inherited name such as `constructor` or `toString` is no threshold.
+      if (!Object.hasOwn(entry.thresholds, name)) {
+        throw new Error(`unknown threshold "${type}.${name}" (${type} thresholds: ${Object.keys(entry.thresholds).join(', ')})`);
+      }
+      values[name] = checkValue(type, name, entry.thresholds[name], value);
+    }
+    result[type] = Object.freeze(values);
+  }
+  return Object.freeze(result)                      ;
+}
+
+/** The overrides for one entry, as the untyped map withThresholds() takes. */
+export function overridesFor(entry          , overrides                                )                                 {
+  return (overrides                                                            )?.[entry.type];
+}
+
+function sameValue(a                            , b                            )          {
+  if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((v, i) => v === b[i]);
+  return a === b;
+}
+
+/** The entry's thresholds that `overrides` moves off the default, or null when none does (no
+ * override, or one equal to the default: that run is the specification's). */
+export function tunedThresholdsOf(entry          , overrides                                )                         {
+  const tuned                                 = {};
+  for (const [name, value] of Object.entries(overridesFor(entry, overrides) ?? {})) {
+    if (!Object.hasOwn(entry.thresholds, name)) continue;
+    const fallback = entry.thresholds[name];
+    if (!sameValue(value, fallback)) tuned[name] = { value, default: fallback };
+  }
+  return Object.keys(tuned).length > 0 ? tuned : null;
+}
+
+/** The tuned thresholds an entry's findings carry: its own, plus its `suppressedBy` entry's
+ * (named `<suppressor>.<threshold>`), since tuning the suppressor changes which of them survive. */
+export function findingTunedThresholds(entry          , overrides                                )                         {
+  const own = tunedThresholdsOf(entry, overrides);
+  const suppressor = entry.suppressedBy ? entries.find((d) => d.type === entry.suppressedBy) : undefined;
+  const bySuppressor = suppressor ? tunedThresholdsOf(suppressor, overrides) : null;
+  if (!suppressor || !bySuppressor) return own;
+  const prefixed = Object.fromEntries(Object.entries(bySuppressor).map(([name, t]) => [`${suppressor.type}.${name}`, t]));
+  return { ...own, ...prefixed };
+}
+
+/** findingTunedThresholds() for the first entry emitting finding type `type`, the entry whose
+ * thresholds its clean-check summary reads. */
+export function tunedThresholdsForType(type        , overrides                                )                         {
+  const entry = ENTRY_BY_TYPE.get(type);
+  return entry ? findingTunedThresholds(entry, overrides) : null;
+}
+
+/** Every tuned detector's overridden thresholds, keyed by entry type, or null when none is tuned. */
+export function tunedDetectors(overrides                                )                                         {
+  const byType                                  = {};
+  for (const entry of entries) {
+    const tuned = tunedThresholdsOf(entry, overrides);
+    if (tuned) byType[entry.type] = tuned;
+  }
+  return Object.keys(byType).length > 0 ? byType : null;
+}
+
+/** The thresholds an entry runs with under `overrides`: its own, with any override merged in. */
+export function effectiveThresholds(entry          , overrides                                )                     {
+  const own = overridesFor(entry, overrides);
+  return own ? { ...entry.thresholds, ...own } : entry.thresholds;
+}
+
+/** detectorCatalog() as a run under `overrides` used it: each row's effective thresholds, plus
+ * `tunedThresholds` on a row an override moved off its defaults. */
+export function tunedDetectorCatalog(overrides                                )                         {
+  return detectorCatalog().map((row, i) => {
+    const entry = entries[i];
+    const tuned = tunedThresholdsOf(entry, overrides);
+    return tuned ? { ...row, thresholds: effectiveThresholds(entry, overrides), tunedThresholds: tuned } : row;
+  });
+}
+
+function formatThresholdValue(value                            )         {
+  return Array.isArray(value) ? `[${value.join(', ')}]` : String(value);
+}
+
+/** "ratioWarn 5 (default 3), minTasksForP95 40 (default 20)". */
+export function describeTunedThresholds(tuned                 )         {
+  return Object.entries(tuned)
+    .map(([name, { value, default: fallback }]) => `${name} ${formatThresholdValue(value)} (default ${formatThresholdValue(fallback)})`)
+    .join(', ');
+}
+
+/** A tuned run's report line after its label: every tuned detector's thresholds, then why its
+ * findings' estimates are uncalibrated. `byType` is tunedDetectors()'s result. */
+export function tunedRunNote(byType                                 )         {
+  const tuned = Object.entries(byType).map(([type, t]) => `${type} ${describeTunedThresholds(t)}`).join('; ');
+  return `${tuned}. Findings from these detectors are marked, and their impact estimates are uncalibrated: the estimates are calibrated against the default thresholds.`;
+}
+
+/** The caveat a tuned finding carries in `validationRequired`: which thresholds produced it and,
+ * when it has an estimate figure (`hasEstimate`), that the figure was never calibrated. */
+export function tunedThresholdsNote(tuned                 , hasEstimate         )         {
+  const label = `Produced with tuned thresholds: ${describeTunedThresholds(tuned)}.`;
+  return hasEstimate
+    ? `${label} Impact estimates are calibrated against the default thresholds, so this finding's estimate is unvalidated.`
+    : label;
+}

package/vendor-core/threshold-summary.js

@@ -1,35 +1,13 @@
-const THRESHOLD_SUMMARIES                         = {
-  spill: 'single-task disk spill above 1 GiB',
-  shuffle: 'shuffle read above the configured minimum byte threshold',
-  skew: 'task duration skew above the configured ratio',
-  gc: 'JVM GC time share above the configured ratio',
-  slowHost: 'a host running 2x+ slower than its peers by mean task duration (per-executor byte/time dimensions use a separate, narrower ratio ladder starting at 1.33x; only those can reach critical on ratio alone)',
-  stageSlowness: 'a stage running far longer than its peers, not attributable to a single slow host',
-  straggler: 'one or more tasks finishing far after the rest of their stage',
-  speculationWaste: 'speculative task attempts that completed after the original',
-  tinyTask: 'median task duration below the configured floor',
-  partitionSizing: 'partition byte size outside the configured target range',
-  stageShape: 'low parallelism, data explosion, or task-count skew relative to core count',
-  stageFailed: 'a stage that failed outright',
-  failures: 'task failures above the configured rate',
-  retryWaste: 'retried task attempts consuming executor time',
-  coldStart: 'executor startup time above the configured floor',
-  incompleteRun: 'an event log missing its terminal ApplicationEnd/job-completion event',
-  utilization: 'core occupancy below the configured floor across the run',
-  memoryUtilization: 'executor heap usage outside the configured band',
-  cacheUtilization: 'cached partitions evicted or spilled to disk',
-  coreLocality: 'task placement missing data-local core assignment',
-  cachingOpportunity: 'a dataset re-read from source multiple times with no cache/persist',
-  jobFailureRate: 'job failure rate above the configured threshold',
-  autoscalingChurn: 'executor add/remove churn above the configured rate',
-  configAudit: 'a Spark conf value outside the recommended range',
-  duplicatePlanSubtree: 'the same physical plan subtree executed more than once',
-  smallFiles: 'output files below the configured target size',
-  overBroadcast: 'a broadcast join above the configured size ceiling',
-  underBroadcast: 'a join below the configured size floor that skipped broadcast',
-  broadcastSizing: 'a broadcast join outside the configured size range in either direction',
-};
+import { ENTRY_BY_TYPE,                         } from './detectors.js';
+import { presentationOf } from './finding-presentation.js';
+import { effectiveThresholds } from './threshold-overrides.js';
 
-export function getThresholdSummary(type        )         {
-  return THRESHOLD_SUMMARIES[type] ?? 'criteria not met';
+/** The clean-check criterion for a finding type, built from the thresholds of the first DETECTORS
+ * entry that emits it (configAudit's four entries share one summary), with `overrides` merged in
+ * when the run was tuned. */
+export function getThresholdSummary(type        , overrides                     )         {
+  const presentation = presentationOf(type);
+  const entry = ENTRY_BY_TYPE.get(type);
+  if (!presentation || !entry) return 'criteria not met';
+  return presentation.thresholdSummary(effectiveThresholds(entry, overrides)         );
 }

package/vendor-core/types.js

@@ -1,3 +1,6 @@
+                                                  
+                                                                
+
                              
                                                          
 
@@ -287,48 +290,9 @@
                             
  
 
-                          
-              
-                           
-               
-                                                                            
-                                                                  
-                                                                            
-                                                                              
-                                                                             
-                                                       
-                           
-                         
-                  
-                                                                        
-                                                                          
-                                                                            
-                                                                         
-                                                                              
-                    
-                          
-                          
-                     
-                      
-                              
-                                  
-                    
-                                          
-                    
-                  
-                          
-                          
-                                                     
-                              
-                                                                                    
-                      
-                                                                        
-                                                                             
-                                                                            
-                                                                         
-                         
-                         
- 
+// `Finding` is a union discriminated on `type`, one member per emitted finding type: see
+// finding-types.ts for each detector's shape and which of its fields are public evidence.
+                                                                                                                  
 
                                                                                      
 

package/vendor-core/vendor/fflate.js

@@ -1,6 +1,6 @@
 // Vendored from fflate@0.8.3 (esm/browser.js), MIT license.
 // https://github.com/101arrowz/fflate — sha256 b7ca4450b19559a1d50eb381adcee94b82449674be4cd17789d9beba7e6122a1
-// Only unzipSync/gunzipSync/zipSync/strToU8/strFromU8 are used by this project (see src/lz4-block.js, src/parser-worker.js).
+// Only Gunzip/UnzipInflate/gunzipSync/strFromU8 are used by this project's source (see src/shs-fetch.ts, src/zip-archive.ts); tests also use the zip writers and strToU8.
 // DEFLATE is a complex format; to read this code, you should probably check the RFC first:
 // https://tools.ietf.org/html/rfc1951
 // You may also wish to take a look at the guide I made about this program:

package/vendor-core/wall-clock.js

@@ -1,15 +1,4 @@
-export function mergeIntervals(intervals                    )                     {
-  if (intervals.length === 0) return [];
-  const sorted = [...intervals].sort((a, b) => a[0] - b[0]);
-  const out                     = [[sorted[0][0], sorted[0][1]]];
-  for (let i = 1; i < sorted.length; i++) {
-    const last = out[out.length - 1];
-    const cur = sorted[i];
-    if (cur[0] <= last[1]) last[1] = Math.max(last[1], cur[1]);
-    else out.push([cur[0], cur[1]]);
-  }
-  return out;
-}
+import { mergeIntervals } from './intervals.js';
 
 export function computeWallClock(app                                                             , stages                                                           )                                                                                     {
   const start = app?.startTime ?? 0;

package/vendor-core/wasted-core-hours.js

@@ -3,11 +3,11 @@
 // core-time that actually ran tasks. Feeds a report widget only, no detector.
 
 import { computeTotalCores } from './core-count.js';
+import { MS_PER_CORE_HOUR } from './format-utils.js';
 
-export const MS_PER_CORE_HOUR         = 3.6e6;
 const TOP_N = 5;
 
-                                 
+                                        
                                 
                                  
                                  

package/vendor-core/zip-archive.js

@@ -0,0 +1,167 @@
+// Random-access zip reader for Spark History Server log archives. Reads the
+// central directory from the archive's tail, then inflates one entry at a time
+// in bounded slices through fflate's streaming UnzipInflate: the source is read
+// a slice at a time and no decompressed entry is ever held whole in memory.
+//
+// Why the central directory and not fflate's forward-scanning `Unzip`: the
+// History Server writes its zip with Java's ZipOutputStream, which leaves
+// every local header's sizes blank and puts them in a data descriptor after
+// the entry. `Unzip` then has to find each entry's end by scanning the
+// compressed bytes for the descriptor signature, and it hands rolling-log
+// parts over in archive order, not the order they must be parsed in. The
+// central directory has the real sizes and lets the caller pick the order.
+import { UnzipInflate, strFromU8 } from './vendor/fflate.js';
+
+                         
+               
+                                                                             
+  
+
+                           
+               
+                                                                              
+                      
+                         
+                            
+ 
+
+const LOCAL_HEADER_SIG = 0x04034b50;
+const CENTRAL_HEADER_SIG = 0x02014b50;
+const EOCD_SIG = 0x06054b50;
+const ZIP64_EOCD_LOCATOR_SIG = 0x07064b50;
+const ZIP64_EOCD_SIG = 0x06064b50;
+const ZIP64_EXTRA_ID = 0x0001;
+const EOCD_MIN_SIZE = 22;
+// The end-of-central-directory record ends with a comment of at most 65535 bytes.
+const EOCD_MAX_SEARCH = EOCD_MIN_SIZE + 0xffff;
+const UINT32_MAX = 0xffffffff;
+
+// fflate's UnzipInflate is untyped vendor JS; this local shape types the call site.
+                         
+                                                                        
+                                                 
+  
+                                                       
+
+const u16 = (d            , i        ) => d[i] | (d[i + 1] << 8);
+const u32 = (d            , i        ) => (d[i] | (d[i + 1] << 8) | (d[i + 2] << 16) | (d[i + 3] << 24)) >>> 0;
+const u64 = (d            , i        ) => u32(d, i) + u32(d, i + 4) * 2 ** 32;
+
+async function readRange(source           , start        , end        )                      {
+  if (start < 0 || end > source.size || start > end) throw new Error('zip structure points outside the file');
+  return new Uint8Array(await source.slice(start, end).arrayBuffer());
+}
+
+/** True when `header` starts with a zip local-file header or an empty archive's EOCD record. */
+export function isZip(header            )          {
+  if (header.length < 4) return false;
+  const sig = u32(header, 0);
+  return sig === LOCAL_HEADER_SIG || sig === EOCD_SIG;
+}
+
+// Locates the central directory via the EOCD record (or its zip64 variant).
+async function readDirectoryLocation(source           )                                                           {
+  const tailStart = Math.max(0, source.size - EOCD_MAX_SEARCH);
+  const tail = await readRange(source, tailStart, source.size);
+  let eocd = tail.length - EOCD_MIN_SIZE;
+  while (eocd >= 0 && u32(tail, eocd) !== EOCD_SIG) eocd--;
+  if (eocd < 0) throw new Error('no end-of-central-directory record');
+
+  const location = { count: u16(tail, eocd + 10), size: u32(tail, eocd + 12), offset: u32(tail, eocd + 16) };
+  const locator = tailStart + eocd - 20;
+  if (locator >= 0) {
+    const loc = await readRange(source, locator, locator + 20);
+    if (u32(loc, 0) === ZIP64_EOCD_LOCATOR_SIG) {
+      const zip64Offset = u64(loc, 8);
+      const z = await readRange(source, zip64Offset, zip64Offset + 56);
+      if (u32(z, 0) !== ZIP64_EOCD_SIG) throw new Error('bad zip64 end-of-central-directory record');
+      return { count: u64(z, 32), size: u64(z, 40), offset: u64(z, 48) };
+    }
+  }
+  return location;
+}
+
+// Reads the zip64 extended-information extra field, which carries (in this
+// order) only the sizes/offset whose 32-bit central-directory field is 0xFFFFFFFF.
+function applyZip64Extra(dir            , extraStart        , extraEnd        , fields                                                              )       {
+  for (let p = extraStart; p + 4 <= extraEnd;) {
+    const id = u16(dir, p), len = u16(dir, p + 2);
+    if (id === ZIP64_EXTRA_ID) {
+      let q = p + 4;
+      for (const key of ['uncompressed', 'compressed', 'offset']         ) {
+        if (fields[key] === UINT32_MAX && q + 8 <= p + 4 + len) { fields[key] = u64(dir, q); q += 8; }
+      }
+      return;
+    }
+    p += 4 + len;
+  }
+}
+
+/** Lists every entry in the archive's central directory, in directory order. */
+export async function listZipEntries(source           )                      {
+  const { count, offset, size } = await readDirectoryLocation(source);
+  const dir = await readRange(source, offset, offset + size);
+  const entries             = [];
+  let p = 0;
+  for (let i = 0; i < count; i++) {
+    if (p + 46 > dir.length || u32(dir, p) !== CENTRAL_HEADER_SIG) throw new Error('bad central-directory header');
+    const nameLength = u16(dir, p + 28), extraLength = u16(dir, p + 30), commentLength = u16(dir, p + 32);
+    const utf8 = (u16(dir, p + 8) & 0x800) !== 0;
+    const nameStart = p + 46, extraStart = nameStart + nameLength;
+    const fields = { uncompressed: u32(dir, p + 24), compressed: u32(dir, p + 20), offset: u32(dir, p + 42) };
+    applyZip64Extra(dir, extraStart, extraStart + extraLength, fields);
+    entries.push({
+      name: strFromU8(dir.subarray(nameStart, extraStart), !utf8),
+      compression: u16(dir, p + 10),
+      compressedSize: fields.compressed,
+      localHeaderOffset: fields.offset,
+    });
+    p = extraStart + extraLength + commentLength;
+  }
+  return entries;
+}
+
+/**
+ * Streams one entry's decompressed bytes to `onChunk`, reading `chunkSize`
+ * compressed bytes at a time and awaiting `onChunk` before reading more, so an
+ * async consumer (an off-thread zstd decoder) applies backpressure. `onRead`
+ * reports compressed bytes consumed, for progress.
+ */
+export async function streamZipEntry(
+  source           ,
+  entry          ,
+  onChunk                                             ,
+  { chunkSize, onRead }                                                         ,
+)                {
+  const header = await readRange(source, entry.localHeaderOffset, entry.localHeaderOffset + 30);
+  if (u32(header, 0) !== LOCAL_HEADER_SIG) throw new Error(`bad local header for "${entry.name}"`);
+  const dataStart = entry.localHeaderOffset + 30 + u16(header, 26) + u16(header, 28);
+  const dataEnd = dataStart + entry.compressedSize;
+  if (dataEnd > source.size) throw new Error(`"${entry.name}" is truncated`);
+  if (entry.compression !== 0 && entry.compression !== 8) {
+    throw new Error(`"${entry.name}" uses unsupported zip compression method ${entry.compression}`);
+  }
+
+  const pending               = [];
+  let inflateError               = null;
+  const inflater = entry.compression === 8 ? new (UnzipInflate                                   )() : null;
+  if (inflater) {
+    inflater.ondata = (err, data) => {
+      if (err) inflateError = err;
+      else if (data.length) pending.push(data);
+    };
+  }
+
+  for (let offset = dataStart; offset < dataEnd;) {
+    const end = Math.min(dataEnd, offset + chunkSize);
+    const slice = await readRange(source, offset, end);
+    offset = end;
+    if (inflater) inflater.push(slice, offset >= dataEnd);
+    else pending.push(slice);
+    if (inflateError) throw inflateError;
+    onRead?.(slice.length);
+    // Inflate output chunks are views that fflate may reuse on the next push,
+    // so each is fully consumed here before the loop reads further.
+    while (pending.length) await onChunk(pending.shift() );
+  }
+}