sparkforensics-server 0.1.0

package/vendor-core/core-time-series.js

@@ -0,0 +1,92 @@
+// Sweep-line "busy cores over time" computation. Pure and side-effect-free so
+// it can run inside parser-worker.js (over retained task launch/finish
+// timestamps) or on the main thread (scaling simulator). One core per task:
+// Spark's default task-to-core mapping.
+//
+// `hypotheticalCores` clamps the concurrent busy-core count to N (the
+// deterministic "utilization at N cores" curve). It does NOT re-schedule tasks;
+// makespan re-estimation is the scaling simulator's job, layered on this signal.
+
+                    
+                 
+                 
+ 
+
+                   
+                 
+               
+               
+ 
+
+function buildStepFunction(intervals            , hypotheticalCores               )            {
+  // Emit +1 at launch, -1 at finish. Skip empty/negative intervals.
+  const events                                      = [];
+  for (const { launch, finish } of intervals) {
+    if (!(finish > launch)) continue;
+    events.push({ t: launch, delta: 1 });
+    events.push({ t: finish, delta: -1 });
+  }
+  // Sort by time; at equal time process -1 before +1 so [launch, finish) is
+  // half-open (a task finishing exactly as another launches does not overlap).
+  events.sort((a, b) => (a.t - b.t) || (a.delta - b.delta));
+
+  // Walk consecutive timestamps, emitting the busy level held on each interval.
+  const segments            = [];
+  let busy = 0;
+  for (let i = 0; i < events.length; i++) {
+    const cur = events[i];
+    busy += cur.delta;
+    const next = events[i + 1];
+    if (!next || next.t === cur.t) continue;
+    const effective = hypotheticalCores != null ? Math.min(busy, hypotheticalCores) : busy;
+    segments.push({ tStart: cur.t, tEnd: next.t, busy: effective });
+  }
+  return segments;
+}
+
+export function computeCoreTimeSeries(intervals            , {
+  bucketBy = 'time',
+  bucketWidthMs = 1000,
+  hypotheticalCores = null,
+}   
+                                  
+                         
+                                    
+  = {})                                                
+               
+                        
+                           
+                         
+                                                                                             
+  {
+  const segments = buildStepFunction(intervals, hypotheticalCores);
+
+  if (bucketBy === 'coreCount') {
+    const histogram           = [];
+    for (const seg of segments) {
+      histogram[seg.busy] = (histogram[seg.busy] ?? 0) + (seg.tEnd - seg.tStart);
+    }
+    // Fill sparse holes with 0 so the array reads as a dense histogram.
+    for (let k = 0; k < histogram.length; k++) if (histogram[k] === undefined) histogram[k] = 0;
+    return { mode: 'coreCount', histogram };
+  }
+
+  // bucketBy === 'time'
+  if (segments.length === 0) {
+    return { mode: 'time', bucketWidthMs, startTime: null, endTime: null, buckets: [] };
+  }
+  const startTime = segments[0].tStart;
+  const endTime = segments[segments.length - 1].tEnd;
+  const buckets                                                                                    = [];
+  for (let tStart = startTime; tStart < endTime; tStart += bucketWidthMs) {
+    const tEnd = tStart + bucketWidthMs;
+    let busyCoreMs = 0;
+    for (const seg of segments) {
+      const lo = Math.max(seg.tStart, tStart);
+      const hi = Math.min(seg.tEnd, tEnd);
+      if (hi > lo) busyCoreMs += seg.busy * (hi - lo);
+    }
+    buckets.push({ tStart, tEnd, busyCoreMs, avgBusyCores: busyCoreMs / bucketWidthMs });
+  }
+  return { mode: 'time', bucketWidthMs, startTime, endTime, buckets };
+}

package/vendor-core/core-usage-locality.js

@@ -0,0 +1,70 @@
+// Stage-granular approximation of core-usage-by-locality over time. See the
+// plan's Task 6 DEVIATION note: per-task locality is not retained, so this
+// distributes each stage's executorRunTime across its wall-clock window,
+// split by localityStats proportions. Not exact; approximate by construction.
+export const LOCALITY_TIERS           = ['PROCESS_LOCAL', 'NODE_LOCAL', 'RACK_LOCAL', 'NO_PREF', 'ANY'];
+
+                         
+                       
+                       
+                           
+                                                        
+ 
+
+                                     
+                         
+                   
+ 
+
+                                    
+                   
+                                   
+ 
+
+export function computeLocalityAreaSeries(
+  stages                 ,
+  { bucketWidthMs = 60_000, tiers = LOCALITY_TIERS }                            = {},
+)                           {
+  const valid = stages.filter(s => (s.completedAt ?? 0) > (s.submittedAt ?? 0) && (s.executorRunTime ?? 0) > 0);
+  if (valid.length === 0) return { labels: [], series: {} };
+  const startTime = valid.reduce((m, s) => Math.min(m, s.submittedAt ?? m), Infinity);
+  const endTime = valid.reduce((m, s) => Math.max(m, s.completedAt ?? m), -Infinity);
+  const nBuckets = Math.max(1, Math.ceil((endTime - startTime) / bucketWidthMs));
+
+  const series                           = {};
+  for (const tier of tiers) series[tier] = new Array(nBuckets).fill(0);
+  const other = new Array(nBuckets).fill(0); // localities not in the fixed tier list
+
+  for (const s of valid) {
+    const completedAt = s.completedAt ?? 0;
+    const submittedAt = s.submittedAt ?? 0;
+    const wall = completedAt - submittedAt;
+    const avgCores = (s.executorRunTime ?? 0) / wall; // core-time / wall-time = avg concurrent cores
+    const total = (s.localityStats ?? []).reduce((a, l) => a + l.count, 0) || 1;
+    const props = new Map((s.localityStats ?? []).map(l => [l.locality, l.count / total]));
+    // spread avgCores over the buckets this stage overlaps, weighted by overlap fraction
+    for (let b = 0; b < nBuckets; b++) {
+      const bStart = startTime + b * bucketWidthMs;
+      const bEnd = bStart + bucketWidthMs;
+      const overlap = Math.min(completedAt, bEnd) - Math.max(submittedAt, bStart);
+      if (overlap <= 0) continue;
+      const frac = overlap / bucketWidthMs; // portion of the bucket this stage covers
+      for (const [loc, p] of props) {
+        const add = avgCores * p * frac;
+        if (series[loc]) series[loc][b] += add;
+        else other[b] += add;
+      }
+    }
+  }
+  if (other.some(v => v > 0)) series.OTHER = other;
+
+  // idle = peak total busy across buckets, minus each bucket's total busy.
+  const totals = new Array(nBuckets).fill(0);
+  for (const tier of Object.keys(series)) for (let b = 0; b < nBuckets; b++) totals[b] += series[tier][b];
+  const peak = totals.reduce((m, v) => Math.max(m, v), 0);
+  series.idle = totals.map(v => Math.max(0, peak - v));
+
+  const labels           = [];
+  for (let b = 0; b < nBuckets; b++) labels.push(startTime + b * bucketWidthMs);
+  return { labels, series };
+}