sparkforensics-mcp 0.2.0 → 0.2.3

package/vendor-core/run-comparison.js

@@ -1,5 +1,6 @@
 import { computeWallClock } from './wall-clock.js';
 import { normalizeDetail } from './detectors.js';
+import { cyrb53 } from './string-hash.js';
 import { captureSnapshot } from './session-snapshot.js';
                                                                                    
                                                              
@@ -35,21 +36,6 @@ export function normalizeStageName(name        )         {
     .trim();
 }
 
-// cyrb53: fast, non-cryptographic, deterministic 53-bit string hash. Not a
-// security boundary; 53 bits keeps collision risk negligible at the per-node,
-// per-stage call volume planTreeIdentity/sqlNodeIdentity put through it.
-function cyrb53(str        , seed = 0)         {
-  let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed;
-  for (let i = 0; i < str.length; i++) {
-    const ch = str.charCodeAt(i);
-    h1 = Math.imul(h1 ^ ch, 2654435761);
-    h2 = Math.imul(h2 ^ ch, 1597334677);
-  }
-  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
-  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
-  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16);
-}
-
 // Bottom-up, order-independent structural identity of a resolved plan tree:
 // each node folds its normalized name/detail with its children's digests
 // (children sorted, so AQE picking a different broadcast side still matches),
@@ -395,6 +381,14 @@ export function buildComparison(
   );
 }
 
+// matchStages' coverage is a Dice coefficient: (2 * pairs.length) / (baseCount
+// + candCount). Below 0.5, more than half of each run's stages went unpaired,
+// so the stage-level rows (stageSkew, baseStages/candStages) mostly show
+// unrelated work side by side rather than the same stage before/after -- the
+// comparison is dominated by guesswork, not genuine pairing. 0.5 is thus the
+// natural midpoint for "more matched than not," not an arbitrary tuning knob.
+const LOW_COVERAGE_THRESHOLD = 0.5;
+
 export function compareRuns(
   baseline                                              ,
   candidate                                              ,
@@ -405,10 +399,21 @@ export function compareRuns(
   // is the normal way to label an A/B experiment, so a mismatch must not block
   // the (matching-free, name-independent) deltas. Surface it as `low` instead.
   const namesDiffer = namesConflict(baseSnap.app, candSnap.app);
+  // Coverage is the other half of the signal: identical names on two runs that
+  // barely share any stages are just as misleading as differing names on two
+  // runs that match well, so either condition alone drops confidence to `low`.
+  const lowCoverage = match.coverage < LOW_COVERAGE_THRESHOLD;
+  const reason = namesDiffer && lowCoverage
+    ? `Run names differ and only ${(match.coverage * 100).toFixed(0)}% of stages matched, so deltas may compare different work.`
+    : namesDiffer
+    ? 'Run names differ, so deltas may compare different work.'
+    : lowCoverage
+    ? `Only ${(match.coverage * 100).toFixed(0)}% of stages matched between runs, so per-stage rows mostly compare unrelated work.`
+    : null;
   return {
     baselineLabel: baseline.label, candidateLabel: candidate.label,
-    confidence: namesDiffer ? 'low' : 'ok',
-    reason: namesDiffer ? 'Run names differ, so deltas may compare different work.' : null,
+    confidence: namesDiffer || lowCoverage ? 'low' : 'ok',
+    reason,
     matchedCoverage: match.coverage,
     metrics: metricDeltas(baseSnap, candSnap),
     findings: findingsDelta(baseSnap, candSnap),

package/vendor-core/shs-fetch.js

@@ -3,7 +3,7 @@ import { createLz4BlockDecoder } from './lz4-block.js';
 import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
 import { createSnappyBlockDecoder } from './snappy-block.js';
 import { buildProxyRequestUrl, isShsErrorCode } from './shs-request.js';
-import { dispatchLine, buildChunkDecoder, emitParseCompletion,                  } from './event-handlers.js';
+import { dispatchLine, buildChunkDecoder, emitParseCompletion,                                   } from './event-handlers.js';
 import { ShsProxyErrorBodySchema } from './shs-schemas.js';
 import { naturalCompare, reassembleRollingEntries } from './rolling-log-reassembly.js';
 
@@ -36,8 +36,14 @@ export function sniffCodec(bytes            )
 // without touching the vendored files.
                                                                            
                                                                                            
+// Like parser-worker.ts's RunOpts.zstdDecoder, but synchronous: decodeEntry never awaits push(),
+// so its result is typed `undefined` (not `void`, which would also accept an async decoder's
+// Promise). Node callers pass cli/native-zstd.ts's createNativeZstdDecoder (nodeArchiveCodecs).
+                                                                                                                            
 
-function decodeEntry(name        , raw            , onChunk                             )       {
+function decodeEntry(
+  name        , raw            , onChunk                             , zstdDecoder                     ,
+)       {
   const codec = sniffCodec(raw);
   if (codec === 'lz4' || name.endsWith('.lz4')) {
     const lz4 = createLz4BlockDecoder(onChunk);
@@ -46,7 +52,7 @@ function decodeEntry(name        , raw            , onChunk
   } else if (codec === 'gz' || name.endsWith('.gz')) {
     new (Gunzip                                   )(onChunk).push(raw, true);
   } else if (codec === 'zstd' || name.endsWith('.zstd') || name.endsWith('.zst')) {
-    new (ZstdDecompress                                   )(onChunk).push(raw, true);
+    (zstdDecoder ? zstdDecoder(onChunk) : new (ZstdDecompress                                   )(onChunk)).push(raw, true);
   } else if (codec === 'snappy' || name.endsWith('.snappy')) {
     const snappy = createSnappyBlockDecoder(onChunk);
     snappy.push(raw);
@@ -138,7 +144,9 @@ export async function runParseFromUrl(
   decodeShsArchive(zipBytes, state, emit);
 }
 
-export function decodeShsArchive(zipBytes            , state             , emit        )       {
+export function decodeShsArchive(
+  zipBytes            , state             , emit        , { zstdDecoder }                                       = {},
+)       {
   let entries                            ;
   try {
     entries = unzipSync(zipBytes);
@@ -165,18 +173,21 @@ export function decodeShsArchive(zipBytes            , state             , emit
   }
 
   const decoder = buildChunkDecoder();
+  const joined               = [];
   let linesProcessed = 0;
   for (const name of names) {
     try {
       decodeEntry(name, entries[name], (bytes) => {
-        for (const line of decoder.decode(bytes)) {
-          dispatchLine(line, state, emit);
+        joined.length = 0;
+        const lines = decoder.decode(bytes, joined);
+        for (let i = 0, j = 0; i < lines.length; i++) {
+          dispatchLine(lines[i], state, emit, joined[j]?.index === i ? joined[j++] : undefined);
           linesProcessed++;
           if (linesProcessed % 2000 === 0) {
             emit({ type: 'progress', pct: null, linesProcessed });
           }
         }
-      });
+      }, zstdDecoder);
     } catch {
       emitShsError(emit, 'invalid-event-log');
       return;

package/vendor-core/shs-load.js

@@ -2,6 +2,7 @@ import { validateShsRequest } from './shs-request.js';
 import { fetchShsEventLog } from './proxy.js';
 import { decodeShsArchive } from './parser-worker.js';
 import { collectViaDispatch } from './cli/collect-run.js';
+import { nodeArchiveCodecs } from './cli/native-zstd.js';
 import { deriveEvidenceAvailability } from './evidence-availability.js';
 import { mcpError } from './mcp-error.js';
                                            
@@ -18,7 +19,7 @@ export const DEFAULT_IDLE_TIMEOUT_MS = envInt('SPARKFORENSICS_SHS_TIMEOUT_MS', 3
 
 function collectShsAppModel(zipBytes            )                                                        {
   return collectViaDispatch(
-    (state, emit) => decodeShsArchive(zipBytes, state, emit),
+    (state, emit) => decodeShsArchive(zipBytes, state, emit, nodeArchiveCodecs),
     (msg) => {
       const m = msg                                       ;
       return mcpError(m?.code ?? 'invalid-event-log', m?.message ?? 'Failed to decode SHS archive.');

package/vendor-core/stage-quantiles.js

@@ -135,13 +135,18 @@ export function finalizeStage(
   const { p50: spillMemP50, p95: spillMemP95, max: spillMemMax } = computeFieldQuantiles(arr, FIELDS.MEM_SPILLED);
   const { p50: spillDiskP50, p95: spillDiskP95, max: spillDiskMax } = computeFieldQuantiles(arr, FIELDS.DISK_SPILLED);
 
-  // Straggler count: tasks with duration > 4 * P50.
+  // Straggler count: tasks with duration > 4 * P50, their summed excess over P50, and the longest
+  // task that isn't one.
   const stragglerThreshold = 4 * p50;
   let stragglerCount = 0;
+  let stragglerExcessMs = 0;
+  let longestNonStragglerMs = 0;
   const taskArrCount = arr.length / FIELDS.STRIDE;
   if (p50 > 0) {
     for (let i = 0; i < taskArrCount; i++) {
-      if (arr[i * FIELDS.STRIDE + FIELDS.DURATION] > stragglerThreshold) stragglerCount++;
+      const duration = arr[i * FIELDS.STRIDE + FIELDS.DURATION];
+      if (duration > stragglerThreshold) { stragglerCount++; stragglerExcessMs += duration - p50; }
+      else if (duration > longestNonStragglerMs) longestNonStragglerMs = duration;
     }
   }
 
@@ -160,9 +165,11 @@ export function finalizeStage(
 
   const data                          = {
     ...stage,
-    hostStats: hostStatsArr, executorStats: executorStatsArr, failureReasons: failureReasonsArr, localityStats: localityStatsArr, stragglerCount,
+    hostStats: hostStatsArr, executorStats: executorStatsArr, failureReasons: failureReasonsArr, localityStats: localityStatsArr, stragglerCount, stragglerExcessMs, longestNonStragglerMs,
     failedTaskSamples,
     peakExecutionMemoryMax,
+    taskActiveMs: computeTaskActiveMs(arr),
+    peakConcurrentTasks: computePeakConcurrentTasks(arr),
     taskDurationP50: p50,
     taskDurationP95: p95,
     taskDurationMax: max,
@@ -178,6 +185,55 @@ export function finalizeStage(
   return { type: 'stage', data };
 }
 
+// Wall-clock time during which at least one of the stage's tasks was running: the union of its
+// [launch, finish) intervals. A stage's submittedAt..completedAt window also covers time it sat
+// open with no task running (waiting for a free slot, or between retried tasks), which no
+// task-level fix can compress. Tasks missing either timestamp are skipped.
+export function computeTaskActiveMs(arr              )         {
+  const taskCount = arr.length / FIELDS.STRIDE;
+  const intervals                     = [];
+  for (let i = 0; i < taskCount; i++) {
+    const launch = arr[i * FIELDS.STRIDE + FIELDS.LAUNCH_TIME];
+    const finish = arr[i * FIELDS.STRIDE + FIELDS.FINISH_TIME];
+    if (launch > 0 && finish > launch) intervals.push([launch, finish]);
+  }
+  intervals.sort((a, b) => a[0] - b[0]);
+  let activeMs = 0, start = -Infinity, end = -Infinity;
+  for (const [launch, finish] of intervals) {
+    if (launch > end) {
+      if (end > start) activeMs += end - start;
+      start = launch;
+      end = finish;
+    } else if (finish > end) {
+      end = finish;
+    }
+  }
+  if (end > start) activeMs += end - start;
+  return activeMs;
+}
+
+// Most tasks running at once, over the same [launch, finish) intervals as computeTaskActiveMs: the
+// slots the stage actually got. A task finishing at the instant another launches frees its slot.
+export function computePeakConcurrentTasks(arr              )         {
+  const taskCount = arr.length / FIELDS.STRIDE;
+  const launches           = [];
+  const finishes           = [];
+  for (let i = 0; i < taskCount; i++) {
+    const launch = arr[i * FIELDS.STRIDE + FIELDS.LAUNCH_TIME];
+    const finish = arr[i * FIELDS.STRIDE + FIELDS.FINISH_TIME];
+    if (launch > 0 && finish > launch) { launches.push(launch); finishes.push(finish); }
+  }
+  launches.sort((a, b) => a - b);
+  finishes.sort((a, b) => a - b);
+  let peak = 0, running = 0, finished = 0;
+  for (const launch of launches) {
+    while (finished < finishes.length && finishes[finished] <= launch) { running--; finished++; }
+    running++;
+    if (running > peak) peak = running;
+  }
+  return peak;
+}
+
 export function computeFieldQuantiles(arr              , fieldIndex        )                                            {
   const taskCount = arr.length / FIELDS.STRIDE;
   if (taskCount === 0) return { p50: 0, p95: 0, max: 0 };

package/vendor-core/string-hash.js

@@ -0,0 +1,15 @@
+// cyrb53: fast, non-cryptographic, deterministic 53-bit string hash. Not a
+// security boundary; 53 bits keeps collision risk negligible at the per-node
+// call volume the plan-identity digests (run-comparison.ts, detectors.ts) put
+// through it.
+export function cyrb53(str        , seed = 0)         {
+  let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed;
+  for (let i = 0; i < str.length; i++) {
+    const ch = str.charCodeAt(i);
+    h1 = Math.imul(h1 ^ ch, 2654435761);
+    h2 = Math.imul(h2 ^ ch, 1597334677);
+  }
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16);
+}

package/vendor-core/types.js

@@ -82,6 +82,15 @@
                            
                            
                                   
+                                                                                      
+                                                                           
+                        
+                                                                                        
+                             
+                                                                                         
+                                 
+                                                                                            
+                               
                                                         
                    
                          
@@ -107,7 +116,6 @@
                                
              
                              
-                                   
                          
  
                                     

package/vendor-core/vendor/fzstd.js

@@ -4,6 +4,17 @@
 // Only the streaming Decompress class is used by this project (see
 // src/parser-worker.js) to inflate Spark event logs written with
 // spark.io.compression.codec=zstd, one block at a time.
+// Local patches, each marked "Local patch" below: Decompress.push loops over
+// frame boundaries instead of recursing; streaming decodes every block into one
+// reused [window | block] buffer (see the note above Decompress) instead of a
+// fresh window per frame and a fresh buffer per block; the sequence loop moves
+// literal runs, non-overlapping matches and window reads longer than CPW_MIN
+// bytes with copyWithin/set instead of a byte loop. On the largest real log
+// they took fzstd from 11.5s to 2.9s in Chrome (5.7s to 2.8s in Node), output
+// byte-identical on all 14 real logs. On 2400 randomly corrupted streams the
+// output and error matched upstream on every one but a corrupt window size
+// that overflows negative: both throw "Invalid typed array length" at the same
+// byte, with a different number in the message (the dev/fuzz-fzstd.mjs check).
 // Some numerical data is initialized as -1 even when it doesn't need initialization to help the JIT infer types
 // aliases for shorter compressed code (most minifers don't do this)
 var ab = ArrayBuffer, u8 = Uint8Array, u16 = Uint16Array, i16 = Int16Array, u32 = Uint32Array, i32 = Int32Array;
@@ -106,7 +117,9 @@ var rzfh = function (dat, w) {
         }
         if (ws > 2145386496)
             err(1);
-        var buf = new u8((w == 1 ? (fss || ws) : w ? 0 : ws) + 12);
+        // Local patch: the streaming Decompress (no `w`) keeps its window in its own buffer (see
+        // the note above Decompress), so none is allocated here.
+        var buf = new u8((w == 1 ? (fss || ws) : 0) + 12);
         buf[0] = 1, buf[4] = 4, buf[8] = 8;
         return {
             b: bt + fsb,
@@ -394,6 +407,13 @@ var dhu4 = function (dat, out, hu) {
     dhu(dat.subarray(bt, bt += dat[4] | (dat[5] << 8)), out.subarray(sz2, sz3), hu);
     dhu(dat.subarray(bt), out.subarray(sz3), hu);
 };
+// Local patch: runs longer than this are copied natively, where a forward byte copy and
+// copyWithin (memmove) agree: a match whose source ends before its destination starts (offset >=
+// length), and a literal run whose destination doesn't pass its source (corrupt input can push
+// the output past the literals). An overlapping match repeats its own output, which only the byte
+// loop does. A source range past the end of its buffer (corrupt input) stays a loop, which reads
+// undefined and so writes 0 there, as upstream does. 16 and 32 measured the same, 8 and 64 slower.
+var CPW_MIN = 16;
 // read Zstandard block
 var rzb = function (dat, st, out) {
     var _a;
@@ -410,7 +430,7 @@ var rzb = function (dat, st, out) {
         st.b = bt + 1;
         if (out) {
             fill(out, dat[bt], st.y, st.y += sz);
-            return out;
+            return st.hv == null ? out : out.subarray(st.y - sz, st.y);
         }
         return fill(new u8(sz), dat[bt]);
     }
@@ -421,7 +441,7 @@ var rzb = function (dat, st, out) {
         if (out) {
             out.set(dat.subarray(bt, ebt), st.y);
             st.y += sz;
-            return out;
+            return st.hv == null ? out : out.subarray(st.y - sz, st.y);
         }
         return slc(dat, bt, ebt);
     }
@@ -548,9 +568,12 @@ var rzb = function (dat, st, out) {
                     else
                         off = st.o[0];
                 }
-                for (var i = 0; i < ll; ++i) {
-                    buf[oubt + i] = buf[spl + i];
-                }
+                if (ll > CPW_MIN && spl + ll <= buf.length && oubt <= spl) // see CPW_MIN
+                    buf.copyWithin(oubt, spl, spl + ll);
+                else
+                    for (var i = 0; i < ll; ++i) {
+                        buf[oubt + i] = buf[spl + i];
+                    }
                 oubt += ll, spl += ll;
                 var stin = oubt - off;
                 if (stin < 0) {
@@ -558,14 +581,34 @@ var rzb = function (dat, st, out) {
                     var bs = st.e + stin;
                     if (len > ml)
                         len = ml;
-                    for (var i = 0; i < len; ++i) {
-                        buf[oubt + i] = st.w[bs + i];
+                    // Local patch: streaming keeps one window buffer for the whole stream (see
+                    // Decompress.push), so history the frame hasn't written yet reads as the 0s
+                    // of upstream's fresh zeroed window.
+                    if (st.hv != null && stin < -st.hv) {
+                        var z = Math.min(len, -st.hv - stin);
+                        fill(buf, 0, oubt, oubt + z);
+                        oubt += z, ml -= z, len -= z, bs += z;
                     }
+                    // Local patch: in bounds only; an out-of-range read stays a loop so it
+                    // still yields 0 (subarray would wrap a negative start).
+                    if (len > CPW_MIN && bs >= 0 && bs + len <= st.w.length && oubt + len <= buf.length) {
+                        if (st.w.buffer === buf.buffer)
+                            st.w.copyWithin(st.e + oubt, bs, bs + len);
+                        else
+                            buf.set(st.w.subarray(bs, bs + len), oubt);
+                    }
+                    else
+                        for (var i = 0; i < len; ++i) {
+                            buf[oubt + i] = st.w[bs + i];
+                        }
                     oubt += len, ml -= len, stin = 0;
                 }
-                for (var i = 0; i < ml; ++i) {
-                    buf[oubt + i] = buf[stin + i];
-                }
+                if (ml > CPW_MIN && oubt - stin >= ml)
+                    buf.copyWithin(oubt, stin, stin + ml);
+                else
+                    for (var i = 0; i < ml; ++i) {
+                        buf[oubt + i] = buf[stin + i];
+                    }
                 oubt += ml;
             }
             if (oubt != spl) {
@@ -575,12 +618,12 @@ var rzb = function (dat, st, out) {
             }
             else
                 oubt = buf.length;
-            if (out)
+            if (out && st.hv == null)
                 st.y += oubt;
             else
-                buf = slc(buf, 0, oubt);
+                buf = buf.subarray(0, oubt); // local patch: buf is this block's own, no copy needed
         }
-        else if (out) {
+        else if (out && st.hv == null) {
             st.y += lss;
             if (spl) {
                 for (var i = 0; i < lss; ++i) {
@@ -589,7 +632,7 @@ var rzb = function (dat, st, out) {
             }
         }
         else if (spl)
-            buf = slc(buf, spl);
+            buf = buf.subarray(spl); // local patch: as above
         st.b = ebt;
         return buf;
     }
@@ -654,6 +697,15 @@ export function decompress(dat, buf) {
     }
     return cct(bufs, ol);
 }
+// Local patch: the streaming Decompress keeps one buffer, this.p, laid out as [window | block]:
+// its first st.e (window size) bytes hold the frame's latest output, and each block is decoded
+// right after them (st.hv tracks how much of the window this frame has written; older positions
+// read as 0, as upstream's fresh zeroed window). A back-reference into an earlier block is then a
+// copy within that one buffer. The block part is zeroed before each block, so a block sees
+// exactly what upstream's fresh block buffer held. Upstream allocated and zeroed a window per
+// frame, a buffer per block, then copied each block out and shifted the window. A chunk passed
+// to ondata is a view of this.p and is only valid until ondata returns: every caller in this
+// project decodes it at once.
 /**
  * Decompressor for Zstandard streamed data
  */
@@ -737,7 +789,22 @@ var Decompress = /*#__PURE__*/ (function () {
                 else
                     this.z = 0;
                 for (;;) {
-                    var blk = rzb(chunk, this.s);
+                    // Local patch: decode into this.p (see the note above Decompress), grown to fit
+                    // the block's declared size, keeping the window.
+                    var st = this.s;
+                    if (st.hv == null)
+                        st.hv = 0;
+                    var hb = st.b, need = Math.max(st.m, (chunk[hb] >> 3) | (chunk[hb + 1] << 5) | (chunk[hb + 2] << 13));
+                    var P = this.p;
+                    if (!P || this.pe != st.e || P.length < st.e + need) {
+                        var np = new u8(st.e + need);
+                        if (P && this.pe == st.e)
+                            np.set(P.subarray(0, st.e));
+                        P = this.p = np, this.pe = st.e;
+                    }
+                    st.w = P, st.y = st.e;
+                    P.fill(0, st.e, st.e + need);
+                    var blk = rzb(chunk, st, P);
                     if (!blk) {
                         if (final)
                             err(5);
@@ -748,8 +815,17 @@ var Decompress = /*#__PURE__*/ (function () {
                     }
                     else {
                         this.ondata(blk, false);
-                        cpw(this.s.w, 0, blk.length);
-                        this.s.w.set(blk, this.s.w.length - blk.length);
+                        // Local patch: upstream's window update, on the window part of this.p. It
+                        // is skipped after a frame's last block, which nothing reads, except
+                        // that a block longer than the window (corrupt input only) still throws
+                        // the RangeError upstream's update does.
+                        if (!st.l) {
+                            P.copyWithin(0, blk.length, st.e);
+                            P.set(blk, st.e - blk.length);
+                            st.hv = Math.min(st.e, st.hv + blk.length);
+                        }
+                        else if (blk.length > st.e)
+                            P.set(blk, st.e - blk.length);
                     }
                     if (this.s.l) {
                         chunk = chunk.subarray(this.s.b);