sparkforensics-mcp 0.2.3 → 0.3.0

package/vendor-core/zip-archive.js

@@ -0,0 +1,167 @@
+// Random-access zip reader for Spark History Server log archives. Reads the
+// central directory from the archive's tail, then inflates one entry at a time
+// in bounded slices through fflate's streaming UnzipInflate: the source is read
+// a slice at a time and no decompressed entry is ever held whole in memory.
+//
+// Why the central directory and not fflate's forward-scanning `Unzip`: the
+// History Server writes its zip with Java's ZipOutputStream, which leaves
+// every local header's sizes blank and puts them in a data descriptor after
+// the entry. `Unzip` then has to find each entry's end by scanning the
+// compressed bytes for the descriptor signature, and it hands rolling-log
+// parts over in archive order, not the order they must be parsed in. The
+// central directory has the real sizes and lets the caller pick the order.
+import { UnzipInflate, strFromU8 } from './vendor/fflate.js';
+
+                         
+               
+                                                                             
+  
+
+                           
+               
+                                                                              
+                      
+                         
+                            
+ 
+
+const LOCAL_HEADER_SIG = 0x04034b50;
+const CENTRAL_HEADER_SIG = 0x02014b50;
+const EOCD_SIG = 0x06054b50;
+const ZIP64_EOCD_LOCATOR_SIG = 0x07064b50;
+const ZIP64_EOCD_SIG = 0x06064b50;
+const ZIP64_EXTRA_ID = 0x0001;
+const EOCD_MIN_SIZE = 22;
+// The end-of-central-directory record ends with a comment of at most 65535 bytes.
+const EOCD_MAX_SEARCH = EOCD_MIN_SIZE + 0xffff;
+const UINT32_MAX = 0xffffffff;
+
+// fflate's UnzipInflate is untyped vendor JS; this local shape types the call site.
+                         
+                                                                        
+                                                 
+  
+                                                       
+
+const u16 = (d            , i        ) => d[i] | (d[i + 1] << 8);
+const u32 = (d            , i        ) => (d[i] | (d[i + 1] << 8) | (d[i + 2] << 16) | (d[i + 3] << 24)) >>> 0;
+const u64 = (d            , i        ) => u32(d, i) + u32(d, i + 4) * 2 ** 32;
+
+async function readRange(source           , start        , end        )                      {
+  if (start < 0 || end > source.size || start > end) throw new Error('zip structure points outside the file');
+  return new Uint8Array(await source.slice(start, end).arrayBuffer());
+}
+
+/** True when `header` starts with a zip local-file header or an empty archive's EOCD record. */
+export function isZip(header            )          {
+  if (header.length < 4) return false;
+  const sig = u32(header, 0);
+  return sig === LOCAL_HEADER_SIG || sig === EOCD_SIG;
+}
+
+// Locates the central directory via the EOCD record (or its zip64 variant).
+async function readDirectoryLocation(source           )                                                           {
+  const tailStart = Math.max(0, source.size - EOCD_MAX_SEARCH);
+  const tail = await readRange(source, tailStart, source.size);
+  let eocd = tail.length - EOCD_MIN_SIZE;
+  while (eocd >= 0 && u32(tail, eocd) !== EOCD_SIG) eocd--;
+  if (eocd < 0) throw new Error('no end-of-central-directory record');
+
+  const location = { count: u16(tail, eocd + 10), size: u32(tail, eocd + 12), offset: u32(tail, eocd + 16) };
+  const locator = tailStart + eocd - 20;
+  if (locator >= 0) {
+    const loc = await readRange(source, locator, locator + 20);
+    if (u32(loc, 0) === ZIP64_EOCD_LOCATOR_SIG) {
+      const zip64Offset = u64(loc, 8);
+      const z = await readRange(source, zip64Offset, zip64Offset + 56);
+      if (u32(z, 0) !== ZIP64_EOCD_SIG) throw new Error('bad zip64 end-of-central-directory record');
+      return { count: u64(z, 32), size: u64(z, 40), offset: u64(z, 48) };
+    }
+  }
+  return location;
+}
+
+// Reads the zip64 extended-information extra field, which carries (in this
+// order) only the sizes/offset whose 32-bit central-directory field is 0xFFFFFFFF.
+function applyZip64Extra(dir            , extraStart        , extraEnd        , fields                                                              )       {
+  for (let p = extraStart; p + 4 <= extraEnd;) {
+    const id = u16(dir, p), len = u16(dir, p + 2);
+    if (id === ZIP64_EXTRA_ID) {
+      let q = p + 4;
+      for (const key of ['uncompressed', 'compressed', 'offset']         ) {
+        if (fields[key] === UINT32_MAX && q + 8 <= p + 4 + len) { fields[key] = u64(dir, q); q += 8; }
+      }
+      return;
+    }
+    p += 4 + len;
+  }
+}
+
+/** Lists every entry in the archive's central directory, in directory order. */
+export async function listZipEntries(source           )                      {
+  const { count, offset, size } = await readDirectoryLocation(source);
+  const dir = await readRange(source, offset, offset + size);
+  const entries             = [];
+  let p = 0;
+  for (let i = 0; i < count; i++) {
+    if (p + 46 > dir.length || u32(dir, p) !== CENTRAL_HEADER_SIG) throw new Error('bad central-directory header');
+    const nameLength = u16(dir, p + 28), extraLength = u16(dir, p + 30), commentLength = u16(dir, p + 32);
+    const utf8 = (u16(dir, p + 8) & 0x800) !== 0;
+    const nameStart = p + 46, extraStart = nameStart + nameLength;
+    const fields = { uncompressed: u32(dir, p + 24), compressed: u32(dir, p + 20), offset: u32(dir, p + 42) };
+    applyZip64Extra(dir, extraStart, extraStart + extraLength, fields);
+    entries.push({
+      name: strFromU8(dir.subarray(nameStart, extraStart), !utf8),
+      compression: u16(dir, p + 10),
+      compressedSize: fields.compressed,
+      localHeaderOffset: fields.offset,
+    });
+    p = extraStart + extraLength + commentLength;
+  }
+  return entries;
+}
+
+/**
+ * Streams one entry's decompressed bytes to `onChunk`, reading `chunkSize`
+ * compressed bytes at a time and awaiting `onChunk` before reading more, so an
+ * async consumer (an off-thread zstd decoder) applies backpressure. `onRead`
+ * reports compressed bytes consumed, for progress.
+ */
+export async function streamZipEntry(
+  source           ,
+  entry          ,
+  onChunk                                             ,
+  { chunkSize, onRead }                                                         ,
+)                {
+  const header = await readRange(source, entry.localHeaderOffset, entry.localHeaderOffset + 30);
+  if (u32(header, 0) !== LOCAL_HEADER_SIG) throw new Error(`bad local header for "${entry.name}"`);
+  const dataStart = entry.localHeaderOffset + 30 + u16(header, 26) + u16(header, 28);
+  const dataEnd = dataStart + entry.compressedSize;
+  if (dataEnd > source.size) throw new Error(`"${entry.name}" is truncated`);
+  if (entry.compression !== 0 && entry.compression !== 8) {
+    throw new Error(`"${entry.name}" uses unsupported zip compression method ${entry.compression}`);
+  }
+
+  const pending               = [];
+  let inflateError               = null;
+  const inflater = entry.compression === 8 ? new (UnzipInflate                                   )() : null;
+  if (inflater) {
+    inflater.ondata = (err, data) => {
+      if (err) inflateError = err;
+      else if (data.length) pending.push(data);
+    };
+  }
+
+  for (let offset = dataStart; offset < dataEnd;) {
+    const end = Math.min(dataEnd, offset + chunkSize);
+    const slice = await readRange(source, offset, end);
+    offset = end;
+    if (inflater) inflater.push(slice, offset >= dataEnd);
+    else pending.push(slice);
+    if (inflateError) throw inflateError;
+    onRead?.(slice.length);
+    // Inflate output chunks are views that fflate may reuse on the next push,
+    // so each is fully consumed here before the loop reads further.
+    while (pending.length) await onChunk(pending.shift() );
+  }
+}

package/vendor-core/zstd-worker-client.js

@@ -0,0 +1,180 @@
+// Parse-worker side of the zstd decompress worker (zstd-worker.ts). Builds a ZstdDecoderFactory
+// for streamFile whose push() ships each compressed read slice to the decompress worker and
+// returns while the worker decodes it, so the parse worker parses slice N's output while the
+// decompress worker decodes slice N+1.
+//
+// Flow control is a window of input slices: push() resolves once fewer than `maxInFlight`
+// slices are unacknowledged, and a slice is acknowledged only when its `consumed` reply is
+// handled, which comes after all of its decoded chunks (each fed to onChunk from the message
+// handler). So at most `maxInFlight` slices' output is ever queued on the parse worker, and a
+// fast decompressor on a slow parse cannot grow memory without bound. The final push resolves
+// only after every chunk was fed, which is what streamFile's callers need before they flush.
+//
+// Buffers move by transfer both ways: push() takes ownership of the slice it is given.
+                                                             
+                                                                           
+
+// Three 512 KiB slices of a ~20x-compressed log keep about 30 MB of output queued at most,
+// and give the decompress worker a slice of slack while the parse worker reads the next one.
+export const MAX_IN_FLIGHT_SLICES = 3;
+
+// The slice of the Worker API this client uses; tests pass a MessagePort adapter.
+                                 
+                                                                      
+                                                                  
+                                             
+                     
+ 
+
+                                           
+                       
+                                                                 
+                                        
+ 
+
+               
+             
+                                      
+                         
+  
+
+// `spawn` starts the decompress worker; it runs at most once, on the first zstd stream, and the
+// worker then serves every later stream (the files of a rolling log) one at a time. When it
+// cannot start (no nested workers, a blocked script, a crash), each stream is decoded by
+// `fallback` on the calling thread instead, the path used before this worker existed.
+export function createWorkerZstdDecoders(
+  spawn                      ,
+  fallback                    ,
+  { maxInFlight = MAX_IN_FLIGHT_SLICES, onFallback }                           = {},
+)                     {
+  let port                        = null;
+  let ready                          = null;
+  let active                = null;
+  let nextId = 0;
+
+  const giveUp = (reason        ) => {
+    port?.terminate?.();
+    port = null;
+    ready = Promise.resolve(false);
+    onFallback?.(reason);
+  };
+
+  const startWorker = ()                   => new Promise((resolve) => {
+    let candidate                ;
+    try {
+      candidate = spawn();
+    } catch (e) {
+      giveUp(`could not start the decompress worker: ${e instanceof Error ? e.message : String(e)}`);
+      resolve(false);
+      return;
+    }
+    let settled = false;
+    const settle = (ok         , reason = '') => {
+      if (settled) return;
+      settled = true;
+      if (ok) port = candidate;
+      else {
+        candidate.terminate?.();
+        giveUp(reason);
+      }
+      resolve(ok);
+    };
+    candidate.onmessage = ({ data }) => {
+      if (data.type === 'ready') settle(true);
+      else if (active && 'id' in data && data.id === active.id) active.onReply(data);
+    };
+    candidate.onerror = (ev) => {
+      // Handled here: left alone, a nested worker's error also reaches the parse worker's own
+      // global handler and from there the page's "Worker crashed" path.
+      ev.preventDefault();
+      const message = ev.message || 'unknown error';
+      if (!settled) {
+        settle(false, `the decompress worker failed to load: ${message}`);
+        return;
+      }
+      const stream = active;
+      giveUp(`the decompress worker crashed: ${message}`);
+      stream?.fail(new Error(`Decompress worker crashed: ${message}`));
+    };
+  });
+
+  return (onChunk) => {
+    const id = nextId++;
+    let started = false;
+    let local                                        = null;
+    let seq = 0;
+    let inFlight = 0;
+    let failure               = null;
+    let wake                      = null;
+
+    const until = (done               ) => new Promise      ((resolve) => {
+      const check = () => {
+        if (!done()) return;
+        wake = null;
+        resolve();
+      };
+      wake = check;
+      check();
+    });
+
+    const stream         = {
+      id,
+      onReply(msg) {
+        if (failure) return;
+        if (msg.type === 'chunk') {
+          try {
+            onChunk(new Uint8Array(msg.bytes, 0, msg.length));
+          } catch (e) {
+            port?.postMessage({ type: 'cancel', id }, []);
+            stream.fail(e instanceof Error ? e : new Error(String(e)));
+          }
+        } else if (msg.type === 'consumed') {
+          inFlight--;
+          wake?.();
+        } else if (msg.type === 'error') {
+          stream.fail(new Error(msg.message));
+        }
+      },
+      fail(err) {
+        if (failure) return;
+        failure = err;
+        if (active === stream) active = null;
+        wake?.();
+      },
+    };
+
+    const begin = async () => {
+      started = true;
+      ready ??= startWorker();
+      if (!(await ready) || !port) {
+        local = fallback(onChunk);
+        return;
+      }
+      active = stream;
+      port.postMessage({ type: 'start', id }, []);
+    };
+
+    return {
+      async push(chunk, final = false) {
+        if (!started) await begin();
+        if (local) return local.push(chunk, final);
+        if (failure) throw failure;
+        // Transfer the slice's own buffer when it spans all of it; copy a view of a larger one.
+        const bytes = chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength
+          ? chunk.buffer               
+          : chunk.slice().buffer;
+        if (!port) throw new Error('Decompress worker is gone');
+        port.postMessage({ type: 'data', id, seq: seq++, bytes, final }, [bytes]);
+        inFlight++;
+        await until(() => failure !== null || inFlight < (final ? 1 : maxInFlight));
+        if (failure) throw failure;
+        if (final && active === stream) active = null;
+      },
+      cancel() {
+        if (local || !started || failure) return;
+        port?.postMessage({ type: 'cancel', id }, []);
+        stream.fail(new Error('Decompression cancelled'));
+      },
+    };
+  };
+}

package/vendor-core/zstd-worker.js

@@ -0,0 +1,103 @@
+// Decompress worker: runs the vendored fzstd off the parse worker's thread, so decompression and
+// NDJSON parsing of a zstd log overlap instead of taking turns. The parse worker spawns it and
+// drives it through zstd-worker-client.ts; both ends of the message protocol live in the types
+// below, and zstd-worker-client.ts documents the flow control.
+import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
+
+// Parse worker -> decompress worker. `seq` numbers a stream's input slices from 0.
+                               
+                                 
+                                                                                 
+                                   
+
+// Decompress worker -> parse worker. Every `chunk` for input slice `seq` is posted before that
+// slice's `consumed`, and a stream's `error` is the last message it gets.
+                             
+                     
+                                                                     
+                                                 
+                                                   
+
+                                                                                       
+
+// fzstd emits one chunk per zstd block (128 KiB at most). Batching them into 1 MiB messages cuts
+// the per-message cost on both threads about eight-fold.
+export const OUTPUT_BATCH_BYTES = 1024 * 1024;
+
+                                                                           
+                                                                                           
+
+// The decompress side of the protocol, transport-free so tests can drive it over a
+// MessageChannel. Holds at most one live stream: `start` replaces it, and a message for any
+// other stream id (one already cancelled or failed) is dropped.
+export function createZstdWorkerHandler(
+  post                ,
+  batchBytes         = OUTPUT_BATCH_BYTES,
+)                                   {
+  let streamId = -1;
+  let decoder                          = null;
+  let batch = new Uint8Array(batchBytes);
+  let batchUsed = 0;
+
+  // Hand `batch` over without copying it again: transfer its whole buffer with the filled
+  // length and start a fresh one. fzstd's chunk is a view of a buffer it reuses, so the copy
+  // into `batch` is the one copy this path cannot avoid.
+  const flush = () => {
+    if (batchUsed === 0) return;
+    const bytes = batch.buffer               ;
+    post({ type: 'chunk', id: streamId, bytes, length: batchUsed }, [bytes]);
+    batch = new Uint8Array(batchBytes);
+    batchUsed = 0;
+  };
+
+  const collect = (chunk            ) => {
+    let offset = 0;
+    while (offset < chunk.length) {
+      const n = Math.min(chunk.length - offset, batchBytes - batchUsed);
+      batch.set(chunk.subarray(offset, offset + n), batchUsed);
+      batchUsed += n;
+      offset += n;
+      if (batchUsed === batchBytes) flush();
+    }
+  };
+
+  const reset = () => {
+    decoder = null;
+    batchUsed = 0;
+  };
+
+  return (msg) => {
+    if (msg.type === 'start') {
+      streamId = msg.id;
+      batchUsed = 0;
+      decoder = new (ZstdDecompress                                   )(collect);
+      return;
+    }
+    if (msg.id !== streamId || !decoder) return;
+    if (msg.type === 'cancel') {
+      reset();
+      return;
+    }
+    try {
+      decoder.push(new Uint8Array(msg.bytes), msg.final);
+      flush();
+    } catch (e) {
+      reset();
+      post({ type: 'error', id: msg.id, message: e instanceof Error ? e.message : String(e) });
+      return;
+    }
+    post({ type: 'consumed', id: msg.id, seq: msg.seq });
+    if (msg.final) reset();
+  };
+}
+
+// ─── Worker message bus (only active when running as a Web Worker) ──────────────
+
+const isWorker = typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope;
+
+if (isWorker) {
+  const scope = self                                         ;
+  const handle = createZstdWorkerHandler((msg, transfer) => scope.postMessage(msg, transfer ?? []));
+  scope.onmessage = ({ data }                                 ) => handle(data);
+  scope.postMessage({ type: 'ready' }                          );
+}