sparkforensics-mcp 0.1.0

package/vendor-core/core-locality-ratio.js

@@ -0,0 +1,54 @@
+// Whole-run core-usage-locality ratio: non-local task share across every
+// stage's `stage.localityStats`. Mirrors wasted-core-hours.js's shape (pure
+// reducer, no detector or view coupling), importable by both the
+// `coreLocality` detector (src/detectors.js) and CoreUsageArea.tsx's
+// stage-breakdown list.
+//
+// Locality classification (Spark, best to worst): PROCESS_LOCAL > NODE_LOCAL
+// > NO_PREF > RACK_LOCAL > ANY. NO_PREF is not a locality failure; it's what
+// shuffle-read stages report because there's no location-preference concept
+// for a shuffle fetch, so it stays in the denominator (diluting the ratio for
+// shuffle-heavy stages, which is correct) but never in the numerator.
+const NON_LOCAL_TIERS = new Set(['RACK_LOCAL', 'ANY']);
+const TOP_N = 5;
+const MIN_TASKS_PER_STAGE = 10;
+
+const EMPTY = { totalTasks: null, nonLocalTasks: null, ratio: null, topStages: [] };
+
+export function computeCoreLocalityRatio(stages                                                                          , { minTasksPerStage = MIN_TASKS_PER_STAGE, topN = TOP_N }                                             = {})                                                                                                                                                                          {
+  if (!Array.isArray(stages) || stages.length === 0) return EMPTY;
+
+  let totalTasks = 0;
+  let nonLocalTasks = 0;
+  const perStage = [];
+
+  for (const stage of stages) {
+    const localityStats = stage?.localityStats;
+    if (!Array.isArray(localityStats) || localityStats.length === 0) continue;
+
+    let stageTotal = 0;
+    let stageNonLocal = 0;
+    for (const { locality, count } of localityStats) {
+      if (!Number.isFinite(count)) continue;
+      stageTotal += count;
+      if (NON_LOCAL_TIERS.has(locality)) stageNonLocal += count;
+    }
+    totalTasks += stageTotal;
+    nonLocalTasks += stageNonLocal;
+
+    if (stageTotal >= minTasksPerStage) {
+      perStage.push({
+        stageId: stage.id,
+        nonLocalTasks: stageNonLocal,
+        taskCount: stageTotal,
+        ratio: stageNonLocal / stageTotal,
+      });
+    }
+  }
+
+  if (totalTasks === 0) return EMPTY;
+
+  const topStages = perStage.sort((a, b) => b.nonLocalTasks - a.nonLocalTasks).slice(0, topN);
+
+  return { totalTasks, nonLocalTasks, ratio: nonLocalTasks / totalTasks, topStages };
+}

package/vendor-core/core-time-series.js

@@ -0,0 +1,92 @@
+// Sweep-line "busy cores over time" computation. Pure and side-effect-free so
+// it can run inside parser-worker.js (over retained task launch/finish
+// timestamps) or on the main thread (scaling simulator). One core per task:
+// Spark's default task-to-core mapping.
+//
+// `hypotheticalCores` clamps the concurrent busy-core count to N (the
+// deterministic "utilization at N cores" curve). It does NOT re-schedule tasks;
+// makespan re-estimation is the scaling simulator's job, layered on this signal.
+
+                    
+                 
+                 
+ 
+
+                   
+                 
+               
+               
+ 
+
+function buildStepFunction(intervals            , hypotheticalCores               )            {
+  // Emit +1 at launch, -1 at finish. Skip empty/negative intervals.
+  const events                                      = [];
+  for (const { launch, finish } of intervals) {
+    if (!(finish > launch)) continue;
+    events.push({ t: launch, delta: 1 });
+    events.push({ t: finish, delta: -1 });
+  }
+  // Sort by time; at equal time process -1 before +1 so [launch, finish) is
+  // half-open (a task finishing exactly as another launches does not overlap).
+  events.sort((a, b) => (a.t - b.t) || (a.delta - b.delta));
+
+  // Walk consecutive timestamps, emitting the busy level held on each interval.
+  const segments            = [];
+  let busy = 0;
+  for (let i = 0; i < events.length; i++) {
+    const cur = events[i];
+    busy += cur.delta;
+    const next = events[i + 1];
+    if (!next || next.t === cur.t) continue;
+    const effective = hypotheticalCores != null ? Math.min(busy, hypotheticalCores) : busy;
+    segments.push({ tStart: cur.t, tEnd: next.t, busy: effective });
+  }
+  return segments;
+}
+
+export function computeCoreTimeSeries(intervals            , {
+  bucketBy = 'time',
+  bucketWidthMs = 1000,
+  hypotheticalCores = null,
+}   
+                                  
+                         
+                                    
+  = {})                                                
+               
+                        
+                           
+                         
+                                                                                             
+  {
+  const segments = buildStepFunction(intervals, hypotheticalCores);
+
+  if (bucketBy === 'coreCount') {
+    const histogram           = [];
+    for (const seg of segments) {
+      histogram[seg.busy] = (histogram[seg.busy] ?? 0) + (seg.tEnd - seg.tStart);
+    }
+    // Fill sparse holes with 0 so the array reads as a dense histogram.
+    for (let k = 0; k < histogram.length; k++) if (histogram[k] === undefined) histogram[k] = 0;
+    return { mode: 'coreCount', histogram };
+  }
+
+  // bucketBy === 'time'
+  if (segments.length === 0) {
+    return { mode: 'time', bucketWidthMs, startTime: null, endTime: null, buckets: [] };
+  }
+  const startTime = segments[0].tStart;
+  const endTime = segments[segments.length - 1].tEnd;
+  const buckets                                                                                    = [];
+  for (let tStart = startTime; tStart < endTime; tStart += bucketWidthMs) {
+    const tEnd = tStart + bucketWidthMs;
+    let busyCoreMs = 0;
+    for (const seg of segments) {
+      const lo = Math.max(seg.tStart, tStart);
+      const hi = Math.min(seg.tEnd, tEnd);
+      if (hi > lo) busyCoreMs += seg.busy * (hi - lo);
+    }
+    buckets.push({ tStart, tEnd, busyCoreMs, avgBusyCores: busyCoreMs / bucketWidthMs });
+  }
+  return { mode: 'time', bucketWidthMs, startTime, endTime, buckets };
+}

package/vendor-core/core-usage-locality.js

@@ -0,0 +1,70 @@
+// Stage-granular approximation of core-usage-by-locality over time. See the
+// plan's Task 6 DEVIATION note: per-task locality is not retained, so this
+// distributes each stage's executorRunTime across its wall-clock window,
+// split by localityStats proportions. Not exact; approximate by construction.
+export const LOCALITY_TIERS           = ['PROCESS_LOCAL', 'NODE_LOCAL', 'RACK_LOCAL', 'NO_PREF', 'ANY'];
+
+                         
+                       
+                       
+                           
+                                                        
+ 
+
+                                     
+                         
+                   
+ 
+
+                                    
+                   
+                                   
+ 
+
+export function computeLocalityAreaSeries(
+  stages                 ,
+  { bucketWidthMs = 60_000, tiers = LOCALITY_TIERS }                            = {},
+)                           {
+  const valid = stages.filter(s => (s.completedAt ?? 0) > (s.submittedAt ?? 0) && (s.executorRunTime ?? 0) > 0);
+  if (valid.length === 0) return { labels: [], series: {} };
+  const startTime = valid.reduce((m, s) => Math.min(m, s.submittedAt ?? m), Infinity);
+  const endTime = valid.reduce((m, s) => Math.max(m, s.completedAt ?? m), -Infinity);
+  const nBuckets = Math.max(1, Math.ceil((endTime - startTime) / bucketWidthMs));
+
+  const series                           = {};
+  for (const tier of tiers) series[tier] = new Array(nBuckets).fill(0);
+  const other = new Array(nBuckets).fill(0); // localities not in the fixed tier list
+
+  for (const s of valid) {
+    const completedAt = s.completedAt ?? 0;
+    const submittedAt = s.submittedAt ?? 0;
+    const wall = completedAt - submittedAt;
+    const avgCores = (s.executorRunTime ?? 0) / wall; // core-time / wall-time = avg concurrent cores
+    const total = (s.localityStats ?? []).reduce((a, l) => a + l.count, 0) || 1;
+    const props = new Map((s.localityStats ?? []).map(l => [l.locality, l.count / total]));
+    // spread avgCores over the buckets this stage overlaps, weighted by overlap fraction
+    for (let b = 0; b < nBuckets; b++) {
+      const bStart = startTime + b * bucketWidthMs;
+      const bEnd = bStart + bucketWidthMs;
+      const overlap = Math.min(completedAt, bEnd) - Math.max(submittedAt, bStart);
+      if (overlap <= 0) continue;
+      const frac = overlap / bucketWidthMs; // portion of the bucket this stage covers
+      for (const [loc, p] of props) {
+        const add = avgCores * p * frac;
+        if (series[loc]) series[loc][b] += add;
+        else other[b] += add;
+      }
+    }
+  }
+  if (other.some(v => v > 0)) series.OTHER = other;
+
+  // idle = peak total busy across buckets, minus each bucket's total busy.
+  const totals = new Array(nBuckets).fill(0);
+  for (const tier of Object.keys(series)) for (let b = 0; b < nBuckets; b++) totals[b] += series[tier][b];
+  const peak = totals.reduce((m, v) => Math.max(m, v), 0);
+  series.idle = totals.map(v => Math.max(0, peak - v));
+
+  const labels           = [];
+  for (let b = 0; b < nBuckets; b++) labels.push(startTime + b * bucketWidthMs);
+  return { labels, series };
+}