sparkforensics-mcp 0.2.2 → 0.2.4

package/vendor-core/vendor/fzstd.js

@@ -4,6 +4,17 @@
 // Only the streaming Decompress class is used by this project (see
 // src/parser-worker.js) to inflate Spark event logs written with
 // spark.io.compression.codec=zstd, one block at a time.
+// Local patches, each marked "Local patch" below: Decompress.push loops over
+// frame boundaries instead of recursing; streaming decodes every block into one
+// reused [window | block] buffer (see the note above Decompress) instead of a
+// fresh window per frame and a fresh buffer per block; the sequence loop moves
+// literal runs, non-overlapping matches and window reads longer than CPW_MIN
+// bytes with copyWithin/set instead of a byte loop. On the largest real log
+// they took fzstd from 11.5s to 2.9s in Chrome (5.7s to 2.8s in Node), output
+// byte-identical on all 14 real logs. On 2400 randomly corrupted streams the
+// output and error matched upstream on every one but a corrupt window size
+// that overflows negative: both throw "Invalid typed array length" at the same
+// byte, with a different number in the message (the dev/fuzz-fzstd.mjs check).
 // Some numerical data is initialized as -1 even when it doesn't need initialization to help the JIT infer types
 // aliases for shorter compressed code (most minifers don't do this)
 var ab = ArrayBuffer, u8 = Uint8Array, u16 = Uint16Array, i16 = Int16Array, u32 = Uint32Array, i32 = Int32Array;
@@ -106,7 +117,9 @@ var rzfh = function (dat, w) {
         }
         if (ws > 2145386496)
             err(1);
-        var buf = new u8((w == 1 ? (fss || ws) : w ? 0 : ws) + 12);
+        // Local patch: the streaming Decompress (no `w`) keeps its window in its own buffer (see
+        // the note above Decompress), so none is allocated here.
+        var buf = new u8((w == 1 ? (fss || ws) : 0) + 12);
         buf[0] = 1, buf[4] = 4, buf[8] = 8;
         return {
             b: bt + fsb,
@@ -394,6 +407,13 @@ var dhu4 = function (dat, out, hu) {
     dhu(dat.subarray(bt, bt += dat[4] | (dat[5] << 8)), out.subarray(sz2, sz3), hu);
     dhu(dat.subarray(bt), out.subarray(sz3), hu);
 };
+// Local patch: runs longer than this are copied natively, where a forward byte copy and
+// copyWithin (memmove) agree: a match whose source ends before its destination starts (offset >=
+// length), and a literal run whose destination doesn't pass its source (corrupt input can push
+// the output past the literals). An overlapping match repeats its own output, which only the byte
+// loop does. A source range past the end of its buffer (corrupt input) stays a loop, which reads
+// undefined and so writes 0 there, as upstream does. 16 and 32 measured the same, 8 and 64 slower.
+var CPW_MIN = 16;
 // read Zstandard block
 var rzb = function (dat, st, out) {
     var _a;
@@ -410,7 +430,7 @@ var rzb = function (dat, st, out) {
         st.b = bt + 1;
         if (out) {
             fill(out, dat[bt], st.y, st.y += sz);
-            return out;
+            return st.hv == null ? out : out.subarray(st.y - sz, st.y);
         }
         return fill(new u8(sz), dat[bt]);
     }
@@ -421,7 +441,7 @@ var rzb = function (dat, st, out) {
         if (out) {
             out.set(dat.subarray(bt, ebt), st.y);
             st.y += sz;
-            return out;
+            return st.hv == null ? out : out.subarray(st.y - sz, st.y);
         }
         return slc(dat, bt, ebt);
     }
@@ -548,9 +568,12 @@ var rzb = function (dat, st, out) {
                     else
                         off = st.o[0];
                 }
-                for (var i = 0; i < ll; ++i) {
-                    buf[oubt + i] = buf[spl + i];
-                }
+                if (ll > CPW_MIN && spl + ll <= buf.length && oubt <= spl) // see CPW_MIN
+                    buf.copyWithin(oubt, spl, spl + ll);
+                else
+                    for (var i = 0; i < ll; ++i) {
+                        buf[oubt + i] = buf[spl + i];
+                    }
                 oubt += ll, spl += ll;
                 var stin = oubt - off;
                 if (stin < 0) {
@@ -558,14 +581,34 @@ var rzb = function (dat, st, out) {
                     var bs = st.e + stin;
                     if (len > ml)
                         len = ml;
-                    for (var i = 0; i < len; ++i) {
-                        buf[oubt + i] = st.w[bs + i];
+                    // Local patch: streaming keeps one window buffer for the whole stream (see
+                    // Decompress.push), so history the frame hasn't written yet reads as the 0s
+                    // of upstream's fresh zeroed window.
+                    if (st.hv != null && stin < -st.hv) {
+                        var z = Math.min(len, -st.hv - stin);
+                        fill(buf, 0, oubt, oubt + z);
+                        oubt += z, ml -= z, len -= z, bs += z;
                     }
+                    // Local patch: in bounds only; an out-of-range read stays a loop so it
+                    // still yields 0 (subarray would wrap a negative start).
+                    if (len > CPW_MIN && bs >= 0 && bs + len <= st.w.length && oubt + len <= buf.length) {
+                        if (st.w.buffer === buf.buffer)
+                            st.w.copyWithin(st.e + oubt, bs, bs + len);
+                        else
+                            buf.set(st.w.subarray(bs, bs + len), oubt);
+                    }
+                    else
+                        for (var i = 0; i < len; ++i) {
+                            buf[oubt + i] = st.w[bs + i];
+                        }
                     oubt += len, ml -= len, stin = 0;
                 }
-                for (var i = 0; i < ml; ++i) {
-                    buf[oubt + i] = buf[stin + i];
-                }
+                if (ml > CPW_MIN && oubt - stin >= ml)
+                    buf.copyWithin(oubt, stin, stin + ml);
+                else
+                    for (var i = 0; i < ml; ++i) {
+                        buf[oubt + i] = buf[stin + i];
+                    }
                 oubt += ml;
             }
             if (oubt != spl) {
@@ -575,12 +618,12 @@ var rzb = function (dat, st, out) {
             }
             else
                 oubt = buf.length;
-            if (out)
+            if (out && st.hv == null)
                 st.y += oubt;
             else
-                buf = slc(buf, 0, oubt);
+                buf = buf.subarray(0, oubt); // local patch: buf is this block's own, no copy needed
         }
-        else if (out) {
+        else if (out && st.hv == null) {
             st.y += lss;
             if (spl) {
                 for (var i = 0; i < lss; ++i) {
@@ -589,7 +632,7 @@ var rzb = function (dat, st, out) {
             }
         }
         else if (spl)
-            buf = slc(buf, spl);
+            buf = buf.subarray(spl); // local patch: as above
         st.b = ebt;
         return buf;
     }
@@ -654,6 +697,15 @@ export function decompress(dat, buf) {
     }
     return cct(bufs, ol);
 }
+// Local patch: the streaming Decompress keeps one buffer, this.p, laid out as [window | block]:
+// its first st.e (window size) bytes hold the frame's latest output, and each block is decoded
+// right after them (st.hv tracks how much of the window this frame has written; older positions
+// read as 0, as upstream's fresh zeroed window). A back-reference into an earlier block is then a
+// copy within that one buffer. The block part is zeroed before each block, so a block sees
+// exactly what upstream's fresh block buffer held. Upstream allocated and zeroed a window per
+// frame, a buffer per block, then copied each block out and shifted the window. A chunk passed
+// to ondata is a view of this.p and is only valid until ondata returns: every caller in this
+// project decodes it at once.
 /**
  * Decompressor for Zstandard streamed data
  */
@@ -737,7 +789,22 @@ var Decompress = /*#__PURE__*/ (function () {
                 else
                     this.z = 0;
                 for (;;) {
-                    var blk = rzb(chunk, this.s);
+                    // Local patch: decode into this.p (see the note above Decompress), grown to fit
+                    // the block's declared size, keeping the window.
+                    var st = this.s;
+                    if (st.hv == null)
+                        st.hv = 0;
+                    var hb = st.b, need = Math.max(st.m, (chunk[hb] >> 3) | (chunk[hb + 1] << 5) | (chunk[hb + 2] << 13));
+                    var P = this.p;
+                    if (!P || this.pe != st.e || P.length < st.e + need) {
+                        var np = new u8(st.e + need);
+                        if (P && this.pe == st.e)
+                            np.set(P.subarray(0, st.e));
+                        P = this.p = np, this.pe = st.e;
+                    }
+                    st.w = P, st.y = st.e;
+                    P.fill(0, st.e, st.e + need);
+                    var blk = rzb(chunk, st, P);
                     if (!blk) {
                         if (final)
                             err(5);
@@ -748,8 +815,17 @@ var Decompress = /*#__PURE__*/ (function () {
                     }
                     else {
                         this.ondata(blk, false);
-                        cpw(this.s.w, 0, blk.length);
-                        this.s.w.set(blk, this.s.w.length - blk.length);
+                        // Local patch: upstream's window update, on the window part of this.p. It
+                        // is skipped after a frame's last block, which nothing reads, except
+                        // that a block longer than the window (corrupt input only) still throws
+                        // the RangeError upstream's update does.
+                        if (!st.l) {
+                            P.copyWithin(0, blk.length, st.e);
+                            P.set(blk, st.e - blk.length);
+                            st.hv = Math.min(st.e, st.hv + blk.length);
+                        }
+                        else if (blk.length > st.e)
+                            P.set(blk, st.e - blk.length);
                     }
                     if (this.s.l) {
                         chunk = chunk.subarray(this.s.b);

package/vendor-core/zstd-worker-client.js

@@ -0,0 +1,180 @@
+// Parse-worker side of the zstd decompress worker (zstd-worker.ts). Builds a ZstdDecoderFactory
+// for streamFile whose push() ships each compressed read slice to the decompress worker and
+// returns while the worker decodes it, so the parse worker parses slice N's output while the
+// decompress worker decodes slice N+1.
+//
+// Flow control is a window of input slices: push() resolves once fewer than `maxInFlight`
+// slices are unacknowledged, and a slice is acknowledged only when its `consumed` reply is
+// handled, which comes after all of its decoded chunks (each fed to onChunk from the message
+// handler). So at most `maxInFlight` slices' output is ever queued on the parse worker, and a
+// fast decompressor on a slow parse cannot grow memory without bound. The final push resolves
+// only after every chunk was fed, which is what streamFile's callers need before they flush.
+//
+// Buffers move by transfer both ways: push() takes ownership of the slice it is given.
+                                                             
+                                                                           
+
+// Three 512 KiB slices of a ~20x-compressed log keep about 30 MB of output queued at most,
+// and give the decompress worker a slice of slack while the parse worker reads the next one.
+export const MAX_IN_FLIGHT_SLICES = 3;
+
+// The slice of the Worker API this client uses; tests pass a MessagePort adapter.
+                                 
+                                                                      
+                                                                  
+                                             
+                     
+ 
+
+                                           
+                       
+                                                                 
+                                        
+ 
+
+               
+             
+                                      
+                         
+  
+
+// `spawn` starts the decompress worker; it runs at most once, on the first zstd stream, and the
+// worker then serves every later stream (the files of a rolling log) one at a time. When it
+// cannot start (no nested workers, a blocked script, a crash), each stream is decoded by
+// `fallback` on the calling thread instead, the path used before this worker existed.
+export function createWorkerZstdDecoders(
+  spawn                      ,
+  fallback                    ,
+  { maxInFlight = MAX_IN_FLIGHT_SLICES, onFallback }                           = {},
+)                     {
+  let port                        = null;
+  let ready                          = null;
+  let active                = null;
+  let nextId = 0;
+
+  const giveUp = (reason        ) => {
+    port?.terminate?.();
+    port = null;
+    ready = Promise.resolve(false);
+    onFallback?.(reason);
+  };
+
+  const startWorker = ()                   => new Promise((resolve) => {
+    let candidate                ;
+    try {
+      candidate = spawn();
+    } catch (e) {
+      giveUp(`could not start the decompress worker: ${e instanceof Error ? e.message : String(e)}`);
+      resolve(false);
+      return;
+    }
+    let settled = false;
+    const settle = (ok         , reason = '') => {
+      if (settled) return;
+      settled = true;
+      if (ok) port = candidate;
+      else {
+        candidate.terminate?.();
+        giveUp(reason);
+      }
+      resolve(ok);
+    };
+    candidate.onmessage = ({ data }) => {
+      if (data.type === 'ready') settle(true);
+      else if (active && 'id' in data && data.id === active.id) active.onReply(data);
+    };
+    candidate.onerror = (ev) => {
+      // Handled here: left alone, a nested worker's error also reaches the parse worker's own
+      // global handler and from there the page's "Worker crashed" path.
+      ev.preventDefault();
+      const message = ev.message || 'unknown error';
+      if (!settled) {
+        settle(false, `the decompress worker failed to load: ${message}`);
+        return;
+      }
+      const stream = active;
+      giveUp(`the decompress worker crashed: ${message}`);
+      stream?.fail(new Error(`Decompress worker crashed: ${message}`));
+    };
+  });
+
+  return (onChunk) => {
+    const id = nextId++;
+    let started = false;
+    let local                                        = null;
+    let seq = 0;
+    let inFlight = 0;
+    let failure               = null;
+    let wake                      = null;
+
+    const until = (done               ) => new Promise      ((resolve) => {
+      const check = () => {
+        if (!done()) return;
+        wake = null;
+        resolve();
+      };
+      wake = check;
+      check();
+    });
+
+    const stream         = {
+      id,
+      onReply(msg) {
+        if (failure) return;
+        if (msg.type === 'chunk') {
+          try {
+            onChunk(new Uint8Array(msg.bytes, 0, msg.length));
+          } catch (e) {
+            port?.postMessage({ type: 'cancel', id }, []);
+            stream.fail(e instanceof Error ? e : new Error(String(e)));
+          }
+        } else if (msg.type === 'consumed') {
+          inFlight--;
+          wake?.();
+        } else if (msg.type === 'error') {
+          stream.fail(new Error(msg.message));
+        }
+      },
+      fail(err) {
+        if (failure) return;
+        failure = err;
+        if (active === stream) active = null;
+        wake?.();
+      },
+    };
+
+    const begin = async () => {
+      started = true;
+      ready ??= startWorker();
+      if (!(await ready) || !port) {
+        local = fallback(onChunk);
+        return;
+      }
+      active = stream;
+      port.postMessage({ type: 'start', id }, []);
+    };
+
+    return {
+      async push(chunk, final = false) {
+        if (!started) await begin();
+        if (local) return local.push(chunk, final);
+        if (failure) throw failure;
+        // Transfer the slice's own buffer when it spans all of it; copy a view of a larger one.
+        const bytes = chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength
+          ? chunk.buffer               
+          : chunk.slice().buffer;
+        if (!port) throw new Error('Decompress worker is gone');
+        port.postMessage({ type: 'data', id, seq: seq++, bytes, final }, [bytes]);
+        inFlight++;
+        await until(() => failure !== null || inFlight < (final ? 1 : maxInFlight));
+        if (failure) throw failure;
+        if (final && active === stream) active = null;
+      },
+      cancel() {
+        if (local || !started || failure) return;
+        port?.postMessage({ type: 'cancel', id }, []);
+        stream.fail(new Error('Decompression cancelled'));
+      },
+    };
+  };
+}

package/vendor-core/zstd-worker.js

@@ -0,0 +1,103 @@
+// Decompress worker: runs the vendored fzstd off the parse worker's thread, so decompression and
+// NDJSON parsing of a zstd log overlap instead of taking turns. The parse worker spawns it and
+// drives it through zstd-worker-client.ts; both ends of the message protocol live in the types
+// below, and zstd-worker-client.ts documents the flow control.
+import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
+
+// Parse worker -> decompress worker. `seq` numbers a stream's input slices from 0.
+                               
+                                 
+                                                                                 
+                                   
+
+// Decompress worker -> parse worker. Every `chunk` for input slice `seq` is posted before that
+// slice's `consumed`, and a stream's `error` is the last message it gets.
+                             
+                     
+                                                                     
+                                                 
+                                                   
+
+                                                                                       
+
+// fzstd emits one chunk per zstd block (128 KiB at most). Batching them into 1 MiB messages cuts
+// the per-message cost on both threads about eight-fold.
+export const OUTPUT_BATCH_BYTES = 1024 * 1024;
+
+                                                                           
+                                                                                           
+
+// The decompress side of the protocol, transport-free so tests can drive it over a
+// MessageChannel. Holds at most one live stream: `start` replaces it, and a message for any
+// other stream id (one already cancelled or failed) is dropped.
+export function createZstdWorkerHandler(
+  post                ,
+  batchBytes         = OUTPUT_BATCH_BYTES,
+)                                   {
+  let streamId = -1;
+  let decoder                          = null;
+  let batch = new Uint8Array(batchBytes);
+  let batchUsed = 0;
+
+  // Hand `batch` over without copying it again: transfer its whole buffer with the filled
+  // length and start a fresh one. fzstd's chunk is a view of a buffer it reuses, so the copy
+  // into `batch` is the one copy this path cannot avoid.
+  const flush = () => {
+    if (batchUsed === 0) return;
+    const bytes = batch.buffer               ;
+    post({ type: 'chunk', id: streamId, bytes, length: batchUsed }, [bytes]);
+    batch = new Uint8Array(batchBytes);
+    batchUsed = 0;
+  };
+
+  const collect = (chunk            ) => {
+    let offset = 0;
+    while (offset < chunk.length) {
+      const n = Math.min(chunk.length - offset, batchBytes - batchUsed);
+      batch.set(chunk.subarray(offset, offset + n), batchUsed);
+      batchUsed += n;
+      offset += n;
+      if (batchUsed === batchBytes) flush();
+    }
+  };
+
+  const reset = () => {
+    decoder = null;
+    batchUsed = 0;
+  };
+
+  return (msg) => {
+    if (msg.type === 'start') {
+      streamId = msg.id;
+      batchUsed = 0;
+      decoder = new (ZstdDecompress                                   )(collect);
+      return;
+    }
+    if (msg.id !== streamId || !decoder) return;
+    if (msg.type === 'cancel') {
+      reset();
+      return;
+    }
+    try {
+      decoder.push(new Uint8Array(msg.bytes), msg.final);
+      flush();
+    } catch (e) {
+      reset();
+      post({ type: 'error', id: msg.id, message: e instanceof Error ? e.message : String(e) });
+      return;
+    }
+    post({ type: 'consumed', id: msg.id, seq: msg.seq });
+    if (msg.final) reset();
+  };
+}
+
+// ─── Worker message bus (only active when running as a Web Worker) ──────────────
+
+const isWorker = typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope;
+
+if (isWorker) {
+  const scope = self                                         ;
+  const handle = createZstdWorkerHandler((msg, transfer) => scope.postMessage(msg, transfer ?? []));
+  scope.onmessage = ({ data }                                 ) => handle(data);
+  scope.postMessage({ type: 'ready' }                          );
+}