sparkforensics-server 0.2.3 → 0.2.4

package/vendor-core/zstd-worker-client.js

@@ -0,0 +1,180 @@
+// Parse-worker side of the zstd decompress worker (zstd-worker.ts). Builds a ZstdDecoderFactory
+// for streamFile whose push() ships each compressed read slice to the decompress worker and
+// returns while the worker decodes it, so the parse worker parses slice N's output while the
+// decompress worker decodes slice N+1.
+//
+// Flow control is a window of input slices: push() resolves once fewer than `maxInFlight`
+// slices are unacknowledged, and a slice is acknowledged only when its `consumed` reply is
+// handled, which comes after all of its decoded chunks (each fed to onChunk from the message
+// handler). So at most `maxInFlight` slices' output is ever queued on the parse worker, and a
+// fast decompressor on a slow parse cannot grow memory without bound. The final push resolves
+// only after every chunk was fed, which is what streamFile's callers need before they flush.
+//
+// Buffers move by transfer both ways: push() takes ownership of the slice it is given.
+                                                             
+                                                                           
+
+// Three 512 KiB slices of a ~20x-compressed log keep about 30 MB of output queued at most,
+// and give the decompress worker a slice of slack while the parse worker reads the next one.
+export const MAX_IN_FLIGHT_SLICES = 3;
+
+// The slice of the Worker API this client uses; tests pass a MessagePort adapter.
+                                 
+                                                                      
+                                                                  
+                                             
+                     
+ 
+
+                                           
+                       
+                                                                 
+                                        
+ 
+
+               
+             
+                                      
+                         
+  
+
+// `spawn` starts the decompress worker; it runs at most once, on the first zstd stream, and the
+// worker then serves every later stream (the files of a rolling log) one at a time. When it
+// cannot start (no nested workers, a blocked script, a crash), each stream is decoded by
+// `fallback` on the calling thread instead, the path used before this worker existed.
+export function createWorkerZstdDecoders(
+  spawn                      ,
+  fallback                    ,
+  { maxInFlight = MAX_IN_FLIGHT_SLICES, onFallback }                           = {},
+)                     {
+  let port                        = null;
+  let ready                          = null;
+  let active                = null;
+  let nextId = 0;
+
+  const giveUp = (reason        ) => {
+    port?.terminate?.();
+    port = null;
+    ready = Promise.resolve(false);
+    onFallback?.(reason);
+  };
+
+  const startWorker = ()                   => new Promise((resolve) => {
+    let candidate                ;
+    try {
+      candidate = spawn();
+    } catch (e) {
+      giveUp(`could not start the decompress worker: ${e instanceof Error ? e.message : String(e)}`);
+      resolve(false);
+      return;
+    }
+    let settled = false;
+    const settle = (ok         , reason = '') => {
+      if (settled) return;
+      settled = true;
+      if (ok) port = candidate;
+      else {
+        candidate.terminate?.();
+        giveUp(reason);
+      }
+      resolve(ok);
+    };
+    candidate.onmessage = ({ data }) => {
+      if (data.type === 'ready') settle(true);
+      else if (active && 'id' in data && data.id === active.id) active.onReply(data);
+    };
+    candidate.onerror = (ev) => {
+      // Handled here: left alone, a nested worker's error also reaches the parse worker's own
+      // global handler and from there the page's "Worker crashed" path.
+      ev.preventDefault();
+      const message = ev.message || 'unknown error';
+      if (!settled) {
+        settle(false, `the decompress worker failed to load: ${message}`);
+        return;
+      }
+      const stream = active;
+      giveUp(`the decompress worker crashed: ${message}`);
+      stream?.fail(new Error(`Decompress worker crashed: ${message}`));
+    };
+  });
+
+  return (onChunk) => {
+    const id = nextId++;
+    let started = false;
+    let local                                        = null;
+    let seq = 0;
+    let inFlight = 0;
+    let failure               = null;
+    let wake                      = null;
+
+    const until = (done               ) => new Promise      ((resolve) => {
+      const check = () => {
+        if (!done()) return;
+        wake = null;
+        resolve();
+      };
+      wake = check;
+      check();
+    });
+
+    const stream         = {
+      id,
+      onReply(msg) {
+        if (failure) return;
+        if (msg.type === 'chunk') {
+          try {
+            onChunk(new Uint8Array(msg.bytes, 0, msg.length));
+          } catch (e) {
+            port?.postMessage({ type: 'cancel', id }, []);
+            stream.fail(e instanceof Error ? e : new Error(String(e)));
+          }
+        } else if (msg.type === 'consumed') {
+          inFlight--;
+          wake?.();
+        } else if (msg.type === 'error') {
+          stream.fail(new Error(msg.message));
+        }
+      },
+      fail(err) {
+        if (failure) return;
+        failure = err;
+        if (active === stream) active = null;
+        wake?.();
+      },
+    };
+
+    const begin = async () => {
+      started = true;
+      ready ??= startWorker();
+      if (!(await ready) || !port) {
+        local = fallback(onChunk);
+        return;
+      }
+      active = stream;
+      port.postMessage({ type: 'start', id }, []);
+    };
+
+    return {
+      async push(chunk, final = false) {
+        if (!started) await begin();
+        if (local) return local.push(chunk, final);
+        if (failure) throw failure;
+        // Transfer the slice's own buffer when it spans all of it; copy a view of a larger one.
+        const bytes = chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength
+          ? chunk.buffer               
+          : chunk.slice().buffer;
+        if (!port) throw new Error('Decompress worker is gone');
+        port.postMessage({ type: 'data', id, seq: seq++, bytes, final }, [bytes]);
+        inFlight++;
+        await until(() => failure !== null || inFlight < (final ? 1 : maxInFlight));
+        if (failure) throw failure;
+        if (final && active === stream) active = null;
+      },
+      cancel() {
+        if (local || !started || failure) return;
+        port?.postMessage({ type: 'cancel', id }, []);
+        stream.fail(new Error('Decompression cancelled'));
+      },
+    };
+  };
+}

package/vendor-core/zstd-worker.js

@@ -0,0 +1,103 @@
+// Decompress worker: runs the vendored fzstd off the parse worker's thread, so decompression and
+// NDJSON parsing of a zstd log overlap instead of taking turns. The parse worker spawns it and
+// drives it through zstd-worker-client.ts; both ends of the message protocol live in the types
+// below, and zstd-worker-client.ts documents the flow control.
+import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
+
+// Parse worker -> decompress worker. `seq` numbers a stream's input slices from 0.
+                               
+                                 
+                                                                                 
+                                   
+
+// Decompress worker -> parse worker. Every `chunk` for input slice `seq` is posted before that
+// slice's `consumed`, and a stream's `error` is the last message it gets.
+                             
+                     
+                                                                     
+                                                 
+                                                   
+
+                                                                                       
+
+// fzstd emits one chunk per zstd block (128 KiB at most). Batching them into 1 MiB messages cuts
+// the per-message cost on both threads about eight-fold.
+export const OUTPUT_BATCH_BYTES = 1024 * 1024;
+
+                                                                           
+                                                                                           
+
+// The decompress side of the protocol, transport-free so tests can drive it over a
+// MessageChannel. Holds at most one live stream: `start` replaces it, and a message for any
+// other stream id (one already cancelled or failed) is dropped.
+export function createZstdWorkerHandler(
+  post                ,
+  batchBytes         = OUTPUT_BATCH_BYTES,
+)                                   {
+  let streamId = -1;
+  let decoder                          = null;
+  let batch = new Uint8Array(batchBytes);
+  let batchUsed = 0;
+
+  // Hand `batch` over without copying it again: transfer its whole buffer with the filled
+  // length and start a fresh one. fzstd's chunk is a view of a buffer it reuses, so the copy
+  // into `batch` is the one copy this path cannot avoid.
+  const flush = () => {
+    if (batchUsed === 0) return;
+    const bytes = batch.buffer               ;
+    post({ type: 'chunk', id: streamId, bytes, length: batchUsed }, [bytes]);
+    batch = new Uint8Array(batchBytes);
+    batchUsed = 0;
+  };
+
+  const collect = (chunk            ) => {
+    let offset = 0;
+    while (offset < chunk.length) {
+      const n = Math.min(chunk.length - offset, batchBytes - batchUsed);
+      batch.set(chunk.subarray(offset, offset + n), batchUsed);
+      batchUsed += n;
+      offset += n;
+      if (batchUsed === batchBytes) flush();
+    }
+  };
+
+  const reset = () => {
+    decoder = null;
+    batchUsed = 0;
+  };
+
+  return (msg) => {
+    if (msg.type === 'start') {
+      streamId = msg.id;
+      batchUsed = 0;
+      decoder = new (ZstdDecompress                                   )(collect);
+      return;
+    }
+    if (msg.id !== streamId || !decoder) return;
+    if (msg.type === 'cancel') {
+      reset();
+      return;
+    }
+    try {
+      decoder.push(new Uint8Array(msg.bytes), msg.final);
+      flush();
+    } catch (e) {
+      reset();
+      post({ type: 'error', id: msg.id, message: e instanceof Error ? e.message : String(e) });
+      return;
+    }
+    post({ type: 'consumed', id: msg.id, seq: msg.seq });
+    if (msg.final) reset();
+  };
+}
+
+// ─── Worker message bus (only active when running as a Web Worker) ──────────────
+
+const isWorker = typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope;
+
+if (isWorker) {
+  const scope = self                                         ;
+  const handle = createZstdWorkerHandler((msg, transfer) => scope.postMessage(msg, transfer ?? []));
+  scope.onmessage = ({ data }                                 ) => handle(data);
+  scope.postMessage({ type: 'ready' }                          );
+}