sparkforensics-server 0.1.0 → 0.2.0

package/vendor-core/cli/budgets.js

@@ -26,10 +26,8 @@ function taskDataTrusted(appModel          )          {
   return entry?.state === 'present';
 }
 
-// `Finding.value` is `number | string` (some detectors, e.g. stageFailed/
-// configAudit, put human-readable text there instead of a magnitude; see
-// types.ts). The 'spill' findings this is used for are always numeric; the
-// typeof guard below narrows without changing behavior for real input.
+// Finding.value is number|string (some detectors put text there); spill findings are
+// always numeric, so the typeof guard narrows without changing behavior for real input.
 function maxFindingValue(catalog           , type        )                {
   const values = catalog
     .filter((f) => f.type === type)
@@ -68,10 +66,8 @@ function checkSkew(appModel          , maxSkewRatio        )               {
   if (stages.length === 0) {
     return { name: 'max-skew', status: 'inconclusive', detail: 'No stage data observed in this event log.' };
   }
-  // Recompute the true ratio per stage (not from the finding catalog): the
-  // skew detector floors its findings at thresholds.ratioWarn (3), so a
-  // budget stricter than that floor could never be enforced by reading
-  // catalog findings alone.
+  // Recompute the true ratio per stage: the skew detector floors findings at
+  // thresholds.ratioWarn (3), so a stricter budget can't be enforced from catalog alone.
   const ratios = stages
     .map((stage) => computeSkewRatio(stage, SKEW_MIN_TASKS_FOR_P95))
     .filter((r) => r !== null)
@@ -96,10 +92,8 @@ function checkFailedTaskRate(appModel          , catalog           , maxPct
   if ((appModel.jobs?.size ?? 0) === 0) {
     return { name: 'max-failed-task-rate', status: 'inconclusive', detail: 'No job data observed in this event log.' };
   }
-  // Jobs completed but the jobFailureRate detector never fired: job failure
-  // rate is below its own 10% info floor (src/detectors.js), so the task
-  // failure rate is implicitly low too. Known v1 limitation: a budget
-  // stricter than that floor cannot be enforced.
+  // jobFailureRate never fired => rate below its 10% info floor, so task failure rate is
+  // implicitly low. v1 limitation: a budget stricter than that floor can't be enforced.
   return { name: 'max-failed-task-rate', status: 'pass', detail: `No job-failure-rate finding: task failure rate is below the detector's reporting floor.` };
 }
 
@@ -125,10 +119,8 @@ function checkRegression(comparison                   , maxRegressionPct
   if (!row || row.direction === 'unavailable' || row.baseline == null || row.delta == null) {
     return { name: 'max-regression', status: 'inconclusive', detail: `Metric "${regressionMetric}" is unavailable for this comparison.` };
   }
-  // A neutral-direction metric (inputBytes, outputBytes, taskCount,
-  // executorsAdded — see NEUTRAL_METRIC_KEYS in run-comparison.ts) measures
-  // workload volume, not performance: an increase isn't a regression, so a
-  // regression budget can't be meaningfully evaluated against it either way.
+  // A neutral-direction metric (inputBytes/outputBytes/taskCount/executorsAdded) measures
+  // workload volume, not performance: an increase isn't a regression, so no direction to check.
   if (row.direction === 'neutral') {
     return { name: 'max-regression', status: 'inconclusive', detail: `Metric "${regressionMetric}" measures workload volume, not performance: it has no regression direction to check.` };
   }
@@ -156,8 +148,7 @@ function checkFailOnIntroduced(comparison                   , band        )
     : { name: 'fail-on-introduced', status: 'pass', detail: `No introduced findings match "${band}".` };
 }
 
-// Shared by both comparison-dependent budgets below: each needs the same
-// "no comparison yet -> inconclusive" fallback instead of actually checking.
+// Both comparison-dependent budgets share the "no comparison yet -> inconclusive" fallback.
 function pushComparisonBudget(
   results                ,
   comparison                               ,
@@ -176,18 +167,13 @@ export function evaluateBudgets({ appModel, catalog, budgets, comparison }
   if (Number.isFinite(budgets.maxSkewRatio)) results.push(checkSkew(appModel, budgets.maxSkewRatio ));
   if (Number.isFinite(budgets.maxFailedTaskRatePct)) results.push(checkFailedTaskRate(appModel, catalog, budgets.maxFailedTaskRatePct ));
   if (Number.isFinite(budgets.minEfficiencyPct)) results.push(checkEfficiency(appModel, budgets.minEfficiencyPct ));
-  // Guarded here (not just at the CLI/mcp-tools call sites) so any caller of
-  // evaluateBudgets() gets this for free: regressionMetric with no
-  // maxRegressionPct would otherwise skip the whole `if` below silently,
-  // reporting nothing at all instead of a visible inconclusive result.
+  // Guarded here so any evaluateBudgets caller benefits: regressionMetric without
+  // maxRegressionPct would otherwise skip the `if` silently, reporting nothing.
   if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
     results.push({ name: 'max-regression', status: 'inconclusive', detail: `regressionMetric "${budgets.regressionMetric}" was set without maxRegressionPct; the regression budget was not evaluated.` });
   } else if (budgets.maxRegressionPct !== undefined) {
-    // `!== undefined`, not `Number.isFinite`: a zero-baseline regression's pct
-    // is itself `Infinity` (see checkRegression), so an "unlimited" budget is a
-    // legitimate finite-typed-as-number input here (CLI's own flag validation
-    // already rejects non-finite --max-regression-pct input, so this only
-    // widens what a direct evaluateBudgets caller, e.g. a test, can express).
+    // `!== undefined`, not Number.isFinite: a zero-baseline regression's pct is Infinity,
+    // so an "unlimited" budget is a legitimate input (the CLI already rejects non-finite flags).
     pushComparisonBudget(results, comparison, 'max-regression',
       (c) => checkRegression(c, budgets.maxRegressionPct , budgets.regressionMetric ?? 'wallClock'));
   }

package/vendor-core/cli/collect-run.js

@@ -1,4 +1,4 @@
-import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { readFileSync, readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
 import { join, basename } from 'node:path';
 import { createState, runParse, runParseFiles, reassembleRollingEntries } from '../parser-worker.js';
 import { createModelCallbacks } from '../model-assembler.js';
@@ -23,8 +23,7 @@ export function emptyAppModel()           {
   };
 }
 
-// Mirrors tests/parser-worker.test.js's fakeFile: the proven File-like shape
-// runParse/runParseFiles need (name, size, slice().arrayBuffer(), arrayBuffer()).
+// File-like shape runParse/runParseFiles need: name, size, slice().arrayBuffer(), arrayBuffer().
 export function nodeFileFromPath(path        )           {
   const bytes = readFileSync(path);
   const u8 = new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
@@ -39,27 +38,53 @@ export function nodeFileFromPath(path        )           {
   };
 }
 
-// Delegates to src/ingest.js's routeMessage: the single source of truth for
-// worker-message-type -> handler-callback wiring, shared by the browser
-// Worker context and this synchronous-in-Node CLI path. No pendingTaskRequests
-// map: the CLI never sends a 'getTaskData' request, so it never receives a
-// 'taskData' message back.
+// Lazy-loading variant for peekLogHeader: only reads the requested byte range from disk,
+// avoiding eager full-file slurps for large logs during directory scans.
+// Used by list_runs local-mode scanning; does NOT have arrayBuffer() since peekLogHeader
+// only needs slice() for bounded reads of the file header.
+export function nodeLazyPeekableFromPath(path        )   
+               
+               
+                                                                             
+  {
+  const size = statSync(path).size;
+  return {
+    name: basename(path),
+    size,
+    slice(start        , end        ) {
+      return {
+        async arrayBuffer()                       {
+          const len = Math.max(0, Math.min(end, size) - start);
+          if (len === 0) return new ArrayBuffer(0);
+          const buf = Buffer.alloc(len);
+          const fd = openSync(path, 'r');
+          try {
+            readSync(fd, buf, 0, len, start);
+          } finally {
+            closeSync(fd);
+          }
+          return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+        },
+      };
+    },
+  };
+}
+
+// routeMessage is the single source of truth for worker-message -> handler wiring, shared by
+// the browser Worker and this Node CLI path. No pendingTaskRequests: the CLI never sends
+// 'getTaskData', so never receives 'taskData' back.
 export function dispatch(msg         , handlers                )       {
   routeMessage(msg                                          , handlers);
 }
 
-function isRollingLogDirectory(dirPath        )          {
+export function isRollingLogDirectory(dirPath        )          {
   const names = readdirSync(dirPath);
   return names.some((n        ) => /^events_\d+_/.test(n));
 }
 
-// Shared ingest scaffold for both consumers of routeMessage's onDone/onError
-// dispatch pattern (this file's collectRun, over a local file/dir, and
-// shs-load.ts's collectShsAppModel, over fetched archive bytes): builds a
-// fresh AppModel wired to the shared handlers, then hands `run` a
-// (state, emit, reject) triple so each caller only supplies its own decode
-// step (and its own error type via `onDecodeError` — a plain Error here, an
-// mcpError over in shs-load.ts).
+// Shared ingest scaffold for both routeMessage consumers (this file's collectRun over a
+// local file/dir, shs-load.ts's collectShsAppModel over fetched bytes): each caller supplies
+// its own decode step and error type via onDecodeError.
 export function collectViaDispatch(
   run                                                                                                                   ,
   onDecodeError                         ,
@@ -96,9 +121,8 @@ export async function collectRun(inputPath        )
         return;
       }
       const files = ordered.map((name) => nodeFileFromPath(join(inputPath, name)));
-      // .catch(reject), not void: an unexpected throw past the parser's own
-      // guards (e.g. in decoder.flush) would otherwise leave this Promise
-      // pending forever and surface only as an unhandled rejection.
+      // .catch(reject), not void: a throw past the parser's guards would otherwise leave
+      // this Promise pending forever, surfacing only as an unhandled rejection.
       runParseFiles(files, state, { emit }).catch(reject);
     } else {
       runParse(nodeFileFromPath(inputPath), state, { emit }).catch(reject);

package/vendor-core/core-count.js

@@ -1,6 +1,4 @@
-// Total cores computation: sum real executor totalCores, or fallback to
-// peakExecutors × configured cores. Used by efficiency model, scaling simulator,
-// wasted core-hours calculation, and utilization/memory-utilization detectors.
+// Total cores: sum executor totalCores, else peakExecutors x configured cores.
 export function computeTotalCores(app                                             , executorsAdded                              )         {
   let total = executorsAdded.reduce((s, e) => s + (e.totalCores ?? 0), 0);
   if (total <= 0) {
@@ -10,19 +8,11 @@ export function computeTotalCores(app
   return total;
 }
 
-// Peak concurrently-alive core count: sweeps add/remove events by timestamp
-// (add contributes +totalCores at its own executorId, remove contributes
-// -totalCores looked up from that same executorId) and tracks the running
-// total's max, rather than summing every addition regardless of overlap.
-// computeTotalCores's cumulative sum overstates capacity under dynamic
-// allocation/executor replacement, since a churned-through executor's cores
-// are never actually concurrent with its replacement's; this is the
-// concurrency-aware sibling used to bound the impact estimator's ceiling
-// (src/occupancy.ts's computeCeiling) against that inflation.
-// Tie-break same-timestamp events by delta ascending, so a removal is applied
-// before a same-instant replacement's addition: without this, a seamless swap
-// (old executor gone exactly when its replacement joins) would momentarily
-// double-count both as concurrent.
+// Peak concurrently-alive core count: sweep add/remove events by timestamp, track running max,
+// rather than summing every addition. A cumulative sum overstates capacity under dynamic
+// allocation (a churned-through executor's cores were never concurrent with its replacement's).
+// Tie-break same-timestamp events by delta ascending so a removal applies before a same-instant
+// addition; else a seamless swap would momentarily double-count both as concurrent.
 function sweepPeak(events                                      )         {
   const sorted = [...events].sort((a, b) => a.time - b.time || a.delta - b.delta);
   let running = 0;
@@ -41,28 +31,36 @@ export function computePeakConcurrentCores(
 )         {
   const coresByExecutor = new Map                ();
   const coreEvents                                       = [];
-  const countEvents                                       = [];
   for (const e of executorsAdded) {
     const cores = e.totalCores ?? 0;
     coresByExecutor.set(e.executorId, cores);
     coreEvents.push({ time: e.timestamp, delta: cores });
-    countEvents.push({ time: e.timestamp, delta: 1 });
   }
   for (const e of executorsRemoved) {
     const cores = coresByExecutor.get(e.executorId) ?? 0;
     coreEvents.push({ time: e.timestamp, delta: -cores });
-    countEvents.push({ time: e.timestamp, delta: -1 });
   }
   const peak = sweepPeak(coreEvents);
   if (peak > 0) return peak;
-  // Real totalCores data is missing/zero for every add event, so the cores
-  // sweep above can't tell us anything. Falling back to
-  // `executorsAdded.length * cores` here would reintroduce the exact
-  // cumulative-historical-additions overcount this function exists to avoid
-  // (a churned-through executor and its replacement both counted, even
-  // though they were never alive at once). Sweep peak *executor count*
-  // instead: still concurrency-aware, just cores-blind.
-  const peakExecutorCount = sweepPeak(countEvents);
+  // totalCores missing/zero for every add, so the cores sweep is blind. Falling back to
+  // executorsAdded.length * cores would reintroduce the cumulative overcount this avoids;
+  // sweep peak executor count instead: concurrency-aware, cores-blind.
+  const peakExecutorCount = computePeakConcurrentExecutorCount(executorsAdded, executorsRemoved);
   const cores = app.resources?.executor?.cores ?? null;
   return cores != null ? peakExecutorCount * cores : 0;
 }
+
+// Peak concurrently-alive executor COUNT (not cores): the denominator memoryUtilization's
+// per-executor waste math (allocatedMB x executor count) needs. Same sweep as
+// computePeakConcurrentCores, over add/remove events rather than cores, so a churned-through
+// executor's slot is never double-counted against its replacement's.
+export function computePeakConcurrentExecutorCount(
+  executorsAdded                                                ,
+  executorsRemoved                                                ,
+)         {
+  const countEvents                                       = [
+    ...executorsAdded.map((e) => ({ time: e.timestamp, delta: 1 })),
+    ...executorsRemoved.map((e) => ({ time: e.timestamp, delta: -1 })),
+  ];
+  return sweepPeak(countEvents);
+}

package/vendor-core/core-locality-ratio.js

@@ -1,14 +1,7 @@
-// Whole-run core-usage-locality ratio: non-local task share across every
-// stage's `stage.localityStats`. Mirrors wasted-core-hours.js's shape (pure
-// reducer, no detector or view coupling), importable by both the
-// `coreLocality` detector (src/detectors.js) and CoreUsageArea.tsx's
-// stage-breakdown list.
-//
-// Locality classification (Spark, best to worst): PROCESS_LOCAL > NODE_LOCAL
-// > NO_PREF > RACK_LOCAL > ANY. NO_PREF is not a locality failure; it's what
-// shuffle-read stages report because there's no location-preference concept
-// for a shuffle fetch, so it stays in the denominator (diluting the ratio for
-// shuffle-heavy stages, which is correct) but never in the numerator.
+// Whole-run core-usage-locality ratio: non-local task share across every stage's localityStats.
+// Locality tiers (Spark, best to worst): PROCESS_LOCAL > NODE_LOCAL > NO_PREF > RACK_LOCAL > ANY.
+// NO_PREF is not a locality failure (shuffle-read stages report it, no location preference for a
+// shuffle fetch), so it stays in the denominator but never the numerator.
 const NON_LOCAL_TIERS = new Set(['RACK_LOCAL', 'ANY']);
 const TOP_N = 5;
 const MIN_TASKS_PER_STAGE = 10;

package/vendor-core/core-time-series.js

@@ -1,11 +1,7 @@
-// Sweep-line "busy cores over time" computation. Pure and side-effect-free so
-// it can run inside parser-worker.js (over retained task launch/finish
-// timestamps) or on the main thread (scaling simulator). One core per task:
-// Spark's default task-to-core mapping.
-//
-// `hypotheticalCores` clamps the concurrent busy-core count to N (the
-// deterministic "utilization at N cores" curve). It does NOT re-schedule tasks;
-// makespan re-estimation is the scaling simulator's job, layered on this signal.
+// Sweep-line "busy cores over time". Pure so it runs in parser-worker (over retained task
+// timestamps) or on the main thread (scaling simulator). One core per task: Spark's default.
+// `hypotheticalCores` clamps the concurrent busy-core count to N; it does NOT re-schedule
+// tasks (makespan re-estimation is the scaling simulator's job, layered on this signal).
 
                     
                  
@@ -19,18 +15,16 @@
  
 
 function buildStepFunction(intervals            , hypotheticalCores               )            {
-  // Emit +1 at launch, -1 at finish. Skip empty/negative intervals.
   const events                                      = [];
   for (const { launch, finish } of intervals) {
     if (!(finish > launch)) continue;
     events.push({ t: launch, delta: 1 });
     events.push({ t: finish, delta: -1 });
   }
-  // Sort by time; at equal time process -1 before +1 so [launch, finish) is
-  // half-open (a task finishing exactly as another launches does not overlap).
+  // At equal time process -1 before +1 so [launch, finish) is half-open (a task finishing
+  // exactly as another launches does not overlap).
   events.sort((a, b) => (a.t - b.t) || (a.delta - b.delta));
 
-  // Walk consecutive timestamps, emitting the busy level held on each interval.
   const segments            = [];
   let busy = 0;
   for (let i = 0; i < events.length; i++) {

package/vendor-core/core-usage-locality.js

@@ -1,7 +1,6 @@
-// Stage-granular approximation of core-usage-by-locality over time. See the
-// plan's Task 6 DEVIATION note: per-task locality is not retained, so this
-// distributes each stage's executorRunTime across its wall-clock window,
-// split by localityStats proportions. Not exact; approximate by construction.
+// Stage-granular approximation of core-usage-by-locality over time. Per-task locality is not
+// retained, so this distributes each stage's executorRunTime across its wall-clock window,
+// split by localityStats proportions. Approximate by construction.
 export const LOCALITY_TIERS           = ['PROCESS_LOCAL', 'NODE_LOCAL', 'RACK_LOCAL', 'NO_PREF', 'ANY'];