sparkforensics-cli 0.1.0

package/vendor-core/ingest.js

@@ -0,0 +1,139 @@
+// Worker/CLI-shared message router: maps a worker (or Node-synchronous,
+// per src/cli/collect-run.js's dispatch()) message to its handler callback.
+// Exported so both contexts run one switch instead of hand-kept-in-sync
+// copies. `pendingTaskRequests` is optional: the CLI path never sends a
+// `getTaskData` request, so it never receives a `taskData` message back.
+// Handler dispatch is optional-chained (`handlers.onXxx?.(...)`), so a
+// caller that adds a new message type here without wiring its handler
+// fails silently (a no-op) rather than throwing.
+
+                                 
+                                       
+                                  
+                                    
+                                  
+                                      
+                                       
+                                  
+                                            
+                                                   
+                                   
+                                    
+ 
+
+                              
+                                
+                               
+ 
+
+export function routeMessage(
+  data                                        ,
+  handlers                ,
+  pendingTaskRequests                                           
+)       {
+  switch (data.type) {
+    case 'progress': handlers.onProgress?.(data); break;
+    case 'app':      handlers.onApp?.(data.data); break;
+    case 'stage':    handlers.onStage?.(data.data); break;
+    case 'sql':      handlers.onSql?.(data.data); break;
+    case 'sqlPlan':  handlers.onSqlPlan?.(data.data); break;
+    case 'executor': handlers.onExecutor?.(data.data); break;
+    case 'job':      handlers.onJob?.(data.data); break;
+    case 'runAggregates': handlers.onRunAggregates?.(data.data); break;
+    case 'stageExecutorMetrics': handlers.onStageExecutorMetrics?.(data.data); break;
+    case 'done': {
+      const { skippedLines } = data;
+      handlers.onDone?.({ skippedLines });
+      break;
+    }
+    case 'error':    handlers.onError?.(data); break;
+    case 'taskData': {
+      const pending = pendingTaskRequests?.get(data.reqId                   );
+      if (pending) {
+        pending.resolve({ metrics: data.metrics, fieldNames: data.fieldNames });
+        pendingTaskRequests .delete(data.reqId                   );
+      }
+      break;
+    }
+  }
+}
+
+                          
+                                   
+                       
+ 
+
+export function createIngestClient()   
+                                                      
+                                                                   
+                                                               
+                                                                
+                                                                                                             
+                        
+  {
+  let worker                = null;
+  const pendingTaskRequests = new Map                                     ();
+  let reqCounter = 0;
+
+  let handlers                 = {};
+
+  // The `new Worker(new URL(...), ...)` expression must appear inline, exactly
+  // like this, for Vite's worker plugin to statically detect and bundle it
+  // (including its own imports, e.g. vendor/fflate.js): routing the URL
+  // through an intermediate variable defeats that detection and leaves the
+  // worker's dependencies unbundled, 404ing at runtime in a production build.
+  function makeWorker() {
+    worker = new Worker(new URL('./parser-worker.js', import.meta.url), { type: 'module' });
+    worker.onmessage = ({ data }) => routeMessage(data, handlers, pendingTaskRequests);
+    worker.onerror = (err) => {
+      handlers.onError?.(`Worker crashed: ${err.message}`);
+      terminate();
+    };
+  }
+
+  function startParse(file      , h                ) {
+    handlers = h;
+    makeWorker();
+    worker .postMessage({ type: 'parse', file });
+  }
+
+  function startParseFromUrl(request         , h                ) {
+    handlers = h;
+    makeWorker();
+    worker .postMessage({ type: 'parseFromUrl', request });
+  }
+
+  function startParseFiles(files         , h                ) {
+    handlers = h;
+    makeWorker();
+    worker .postMessage({ type: 'parseFiles', files });
+  }
+
+  function requestTaskData(stageId        )                          {
+    return new Promise         ((resolve, reject) => {
+      const reqId = String(reqCounter++);
+      pendingTaskRequests.set(reqId, { resolve, reject });
+      worker .postMessage({ type: 'getTaskData', stageId, reqId });
+    })                           ;
+  }
+
+  async function prefetchFlaggedStages(flaggedStageIds          )                                                       {
+    const results = await Promise.all(
+      flaggedStageIds.map(stageId =>
+        requestTaskData(stageId).then(data => ({ stageId, ...data }))
+      )
+    );
+    return results;
+  }
+
+  function terminate() {
+    worker?.terminate();
+    worker = null;
+    for (const { reject } of pendingTaskRequests.values()) {
+      reject(new Error('Worker terminated'));
+    }
+    pendingTaskRequests.clear();
+  }
+
+  return { startParse, startParseFromUrl, startParseFiles, requestTaskData, prefetchFlaggedStages, terminate };
+}

package/vendor-core/job-groups.js

@@ -0,0 +1,30 @@
+// §8 Concurrent-job-group reliability guard (SparkLens JobOverlapAnalyzer).
+// Groups jobs by SQL execution id; jobs with none form their own singleton
+// group. If two DIFFERENT groups' [submissionTime, completionTime] intervals
+// overlap, wall-clock-based estimates (scaling simulator, efficiency model)
+// become unreliable. Jobs WITHIN a group overlapping (AQE/multi-stage) is
+// normal and not the signal.
+export function checkConcurrentJobGroups(jobs                                                                                                                             )                                                                        {
+  const list = jobs ? [...jobs.values()] : [];
+  // Build one interval per group: [min submission, max completion].
+  const groups = new Map(); // key -> { start, end }
+  let singletonSeq = 0;
+  for (const j of list) {
+    if (j.submissionTime == null || j.completionTime == null) continue;
+    const key = j.sqlExecutionId != null ? `sql:${j.sqlExecutionId}` : `job:${j.id ?? singletonSeq++}`;
+    const g = groups.get(key);
+    if (!g) groups.set(key, { start: j.submissionTime, end: j.completionTime });
+    else { g.start = Math.min(g.start, j.submissionTime); g.end = Math.max(g.end, j.completionTime); }
+  }
+
+  const entries = [...groups.entries()].sort((a, b) => a[1].start - b[1].start);
+  const overlappingGroupIds                     = [];
+  for (let i = 1; i < entries.length; i++) {
+    for (let k = 0; k < i; k++) {
+      const [aKey, a] = entries[k];
+      const [bKey, b] = entries[i];
+      if (b.start < a.end && a.start < b.end) overlappingGroupIds.push([aKey, bKey]                    );
+    }
+  }
+  return { wallClockReliable: overlappingGroupIds.length === 0, overlappingGroupIds };
+}

package/vendor-core/load-vendored.js

@@ -0,0 +1,24 @@
+// Plain JS (like proxy.js): scripts/vendor-core.mjs copies it byte-for-byte
+// into vendor-core/, so this exact file exists at both
+// vendor-core/load-vendored.js (published install) and
+// core/src/load-vendored.js (monorepo dev). Each published package's bin
+// entry point bootstraps by locating *this* file first (a tiny fixed-name
+// existsSync/join check, unavoidably duplicated per entry point since
+// nothing can resolve it for them), then uses the exports below for every
+// other core module, so the actual resolution logic lives in exactly one
+// place instead of being re-implemented per entry point.
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+// moduleName is a path relative to core/src without extension, e.g.
+// 'cli/collect-run' or 'shs-load'. Pass srcExt: 'js' for modules that are
+// already plain JS in core/src (e.g. 'proxy') rather than TypeScript.
+export function resolveVendored(pkgDir, moduleName, { srcExt = 'ts' } = {}) {
+  const vendored = join(pkgDir, 'vendor-core', `${moduleName}.js`);
+  return existsSync(vendored) ? vendored : join(pkgDir, '..', 'core', 'src', `${moduleName}.${srcExt}`);
+}
+
+export async function loadVendored(pkgDir, moduleName, opts) {
+  return import(pathToFileURL(resolveVendored(pkgDir, moduleName, opts)).href);
+}

package/vendor-core/lz4-block.js

@@ -0,0 +1,135 @@
+const MAGIC = [76, 90, 52, 66, 108, 111, 99, 107]; // "LZ4Block"
+
+function readInt32LE(bytes            , offset        )         {
+  return (bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 24)) >>> 0;
+}
+
+// Standard LZ4 block decompression (the same algorithm used by every LZ4
+// implementation: only the outer per-block framing above is Spark-specific).
+function decompressLz4Sequence(input            , outSize        )             {
+  const out = new Uint8Array(outSize);
+  let ip = 0, op = 0;
+  const n = input.length;
+  while (ip < n) {
+    const token = input[ip++];
+    let literalLength = token >> 4;
+    if (literalLength === 15) {
+      let b;
+      do { b = input[ip++]; literalLength += b; } while (b === 255);
+    }
+    out.set(input.subarray(ip, ip + literalLength), op);
+    ip += literalLength;
+    op += literalLength;
+    if (ip >= n) break; // final sequence has no match part
+    const offset = input[ip] | (input[ip + 1] << 8);
+    ip += 2;
+    let matchLength = token & 0x0f;
+    if (matchLength === 15) {
+      let b;
+      do { b = input[ip++]; matchLength += b; } while (b === 255);
+    }
+    matchLength += 4; // minimum match length
+    let matchPos = op - offset;
+    for (let i = 0; i < matchLength; i++) out[op++] = out[matchPos++];
+  }
+  return out;
+}
+
+// Streaming counterpart to decodeLz4Block: push arbitrary byte slices, and each
+// fully-received LZ4Block block is decompressed and handed to `onChunk` as it
+// completes, so the decompressed output is never fully buffered (bounded to one
+// block at a time). Partial trailing bytes are retained until the next push.
+// `onChunk` must consume its argument synchronously (it aliases the internal
+// buffer for RAW blocks and is not retained across the next push).
+export function createLz4BlockDecoder(
+  onChunk                             ,
+)                                                 {
+  let buf             = new Uint8Array(0);
+  return {
+    push(chunk            ) {
+      if (buf.length === 0) {
+        buf = chunk;
+      } else if (chunk.length) {
+        const merged = new Uint8Array(buf.length + chunk.length);
+        merged.set(buf); merged.set(chunk, buf.length);
+        buf = merged;
+      }
+      let pos = 0;
+      while (buf.length - pos >= 21) {
+        for (let i = 0; i < 8; i++) {
+          if (buf[pos + i] !== MAGIC[i]) {
+            throw new Error(`Not a Spark LZ4Block stream: bad magic at offset ${pos}.`);
+          }
+        }
+        const method = buf[pos + 8] & 0xf0;
+        const compressedLength = readInt32LE(buf, pos + 9);
+        const decompressedLength = readInt32LE(buf, pos + 13);
+        const bodyStart = pos + 21;
+        if (buf.length - bodyStart < compressedLength) break; // block not fully arrived yet
+        const body = buf.subarray(bodyStart, bodyStart + compressedLength);
+
+        let out;
+        if (method === 0x10) {
+          out = body;
+        } else if (method === 0x20) {
+          out = decompressLz4Sequence(body, decompressedLength);
+        } else {
+          throw new Error(`Unknown LZ4Block method 0x${method.toString(16)} at offset ${pos}.`);
+        }
+        if (out.length !== decompressedLength) {
+          throw new Error(`LZ4Block length mismatch at offset ${pos}: expected ${decompressedLength}, got ${out.length}.`);
+        }
+        onChunk(out);
+        pos = bodyStart + compressedLength;
+      }
+      // Copy (not subarray) the unconsumed tail so the large read-slice buffer
+      // can be garbage-collected instead of being pinned by a view.
+      buf = pos > 0 ? buf.slice(pos) : buf;
+    },
+    end() {
+      if (buf.length !== 0) {
+        throw new Error(`Trailing ${buf.length} undecoded bytes in LZ4Block stream.`);
+      }
+    },
+  };
+}
+
+export function decodeLz4Block(bytes            )             {
+  const chunks               = [];
+  let pos = 0;
+  const n = bytes.length;
+  while (pos < n) {
+    for (let i = 0; i < 8; i++) {
+      if (bytes[pos + i] !== MAGIC[i]) {
+        throw new Error(`Not a Spark LZ4Block stream: bad magic at offset ${pos}.`);
+      }
+    }
+    const token = bytes[pos + 8];
+    const method = token & 0xf0;
+    const compressedLength = readInt32LE(bytes, pos + 9);
+    const decompressedLength = readInt32LE(bytes, pos + 13);
+    // checksum at pos+17..pos+20 (xxhash32 of decompressed data), not verified
+    const bodyStart = pos + 21;
+    const body = bytes.subarray(bodyStart, bodyStart + compressedLength);
+
+    let chunk;
+    if (method === 0x10) {
+      chunk = body;
+    } else if (method === 0x20) {
+      chunk = decompressLz4Sequence(body, decompressedLength);
+    } else {
+      throw new Error(`Unknown LZ4Block method 0x${method.toString(16)} at offset ${pos}.`);
+    }
+    if (chunk.length !== decompressedLength) {
+      throw new Error(`LZ4Block length mismatch at offset ${pos}: expected ${decompressedLength}, got ${chunk.length}.`);
+    }
+    chunks.push(chunk);
+    pos = bodyStart + compressedLength;
+  }
+
+  const total = chunks.reduce((sum, c) => sum + c.length, 0);
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const c of chunks) { out.set(c, offset); offset += c.length; }
+  return out;
+}

package/vendor-core/mcp-error.js

@@ -0,0 +1,3 @@
+export function mcpError(code        , message        )                           {
+  return Object.assign(new Error(message), { code });
+}

package/vendor-core/mcp-server-factory.js

@@ -0,0 +1,115 @@
+import { z } from 'zod';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+                                                                         
+import {
+  resolveOrCreateRun, diagnoseRun, getRunSummary, compareRuns, getFindingEvidence, evaluateBudgetsForRun,
+} from './mcp-tools.js';
+
+const sourceSchema = z.union([
+  z.object({ path: z.string() }),
+  z.object({ shsBaseUrl: z.string(), appId: z.string(), attemptId: z.string().optional() }),
+]);
+
+const runRefSchema = { source: sourceSchema.optional(), runId: z.string().optional() };
+const secondRunRefSchema = { sourceB: sourceSchema.optional(), runIdB: z.string().optional() };
+const formatSchema = { format: z.enum(['json', 'md']).optional() };
+
+function toCallToolResult(text        , structuredContent                         )                 {
+  return { content: [{ type: 'text'         , text }], structuredContent };
+}
+
+function toolErrorResult(error                                    )                 {
+  return {
+    isError: true,
+    content: [{ type: 'text'         , text: error.message }],
+    structuredContent: { code: error.code ?? 'access-or-upstream-failure' },
+  };
+}
+
+// diagnoseRun/compareRuns may additionally carry a `markdown` field (only
+// when the caller asked for `format: 'md'`); that field never belongs in
+// structuredContent (its shape must stay stable regardless of `format`), so
+// it's stripped here, and — when present — becomes content[0].text instead
+// of the usual JSON.stringify.
+function toolResultWithMarkdown                  (promise                                    )                          {
+  return promise.then(
+    (value) => {
+      const { markdown, ...structuredContent } = value;
+      return toCallToolResult(markdown !== undefined ? markdown : JSON.stringify(structuredContent), structuredContent                           );
+    },
+    toolErrorResult,
+  );
+}
+
+// Counterpart for the 3 tools that never produce a markdown field: its own
+// mapping (not a delegation through toolResultWithMarkdown), so there's no
+// need to cast T to pretend it might carry a `markdown` field it never does.
+function toolResult                  (promise            )                          {
+  return promise.then((value) => toCallToolResult(JSON.stringify(value), value                           ), toolErrorResult);
+}
+
+export function createMcpServer()            {
+  const server = new McpServer({ name: 'sparkforensics', version: '1.0.0' });
+
+  server.registerTool('diagnose_run', {
+    description: 'Diagnose a Spark run: thresholded findings with remediation text, an impact-ranked fix recommendation rollup, and clean-check status.',
+    inputSchema: {
+      ...runRefSchema, redact: z.boolean().optional(),
+      include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
+      impactBand: z.array(z.string()).optional(), type: z.array(z.string()).optional(), stageId: z.number().int().optional(),
+      ...formatSchema,
+    },
+  }, ({ source, runId, redact, include, format, impactBand, type, stageId }) => toolResultWithMarkdown(
+    resolveOrCreateRun({ source, runId }).then(({ runId: id }) =>
+      diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId })),
+  ));
+
+  server.registerTool('get_run_summary', {
+    description: 'App/stage/job/sql counts and duration for a run, no findings.',
+    inputSchema: { ...runRefSchema, redact: z.boolean().optional() },
+  }, ({ source, runId, redact }) => toolResult(
+    resolveOrCreateRun({ source, runId }).then(({ runId: id }) => getRunSummary(id, { redact })),
+  ));
+
+  server.registerTool('compare_runs', {
+    description: 'Compare two runs: categorized findings delta and metric deltas.',
+    inputSchema: {
+      runIdA: z.string().optional(), sourceA: sourceSchema.optional(),
+      ...secondRunRefSchema,
+      redact: z.boolean().optional(),
+      ...formatSchema,
+    },
+  }, ({ runIdA, sourceA, runIdB, sourceB, redact, format }) => toolResultWithMarkdown(
+    compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md' }),
+  ));
+
+  server.registerTool('evaluate_budgets', {
+    description: 'Evaluate a run (optionally against a second run for regression budgets) against pass/fail thresholds.',
+    inputSchema: {
+      source: sourceSchema.optional().describe('The run to evaluate. Also the regression baseline when sourceB/runIdB is given.'),
+      runId: runRefSchema.runId.describe('Same as `source`, referencing an already-resolved run by id.'),
+      maxRuntimeMs: z.number().optional(), maxSpillGb: z.number().optional(), maxSkewRatio: z.number().optional(),
+      maxFailedTaskRatePct: z.number().optional(), minEfficiencyPct: z.number().optional(),
+      sourceB: sourceSchema.optional().describe('Optional candidate run, compared against source/runId as the regression baseline (maxRegressionPct/failOnIntroduced).'),
+      runIdB: secondRunRefSchema.runIdB.describe('Same as `sourceB`, referencing an already-resolved run by id.'),
+      maxRegressionPct: z.number().optional(), regressionMetric: z.string().optional(), failOnIntroduced: z.string().optional(),
+    },
+  }, ({
+    source, runId, runIdB, sourceB,
+    maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct,
+    maxRegressionPct, regressionMetric, failOnIntroduced,
+  }) => toolResult(evaluateBudgetsForRun(
+    { source, runId },
+    { maxRuntimeMs, maxSpillGb, maxSkewRatio, maxFailedTaskRatePct, minEfficiencyPct, maxRegressionPct, regressionMetric, failOnIntroduced },
+    (runIdB || sourceB) ? { runId: runIdB, source: sourceB } : undefined,
+  )));
+
+  server.registerTool('get_finding_evidence', {
+    description: 'Raw evidence bundle backing one finding, for drill-down after diagnose_run.',
+    inputSchema: { runId: z.string(), findingId: z.string(), redact: z.boolean().optional() },
+  }, ({ runId, findingId, redact }) => toolResult(
+    Promise.resolve().then(() => getFindingEvidence(runId, findingId, { redact })),
+  ));
+
+  return server;
+}