sparkforensics-server 0.6.0 → 0.8.1

package/vendor-core/aqe-skew.js

@@ -0,0 +1,347 @@
+// Why AQE skew-join handling did or did not act on a stage's join, read from the execution's final
+// plan and the effective Spark conf. Spark's OptimizeSkewedJoin (apache/spark v3.5.0,
+// sql/core/.../adaptive/OptimizeSkewedJoin.scala) splits a skewed shuffle partition only when the
+// join is a sort-merge or shuffled-hash join over shuffles Spark added for it, the join type lets
+// the skewed side be split, the partition is over both the size threshold and the factor times the
+// median, and the split adds no shuffle above the join (unless forceOptimizeSkewedJoin). The
+// final plan shows a split (`skew=true`, `AQEShuffleRead skewed`) and the shape the rule matches,
+// so the first condition that fails is the reason. The cases are in
+// user-guide/understanding-findings.md.
+                                           
+                                                                   
+import { formatBytes } from './format-utils.js';
+import { walkPlanTree } from './plan-tree-walk.js';
+import { codeFix, decreaseConf, setConf } from './remediation.js';
+import { parseSparkBytes, sparkConfDefault } from './spark-conf.js';
+
+export const SKEW_THRESHOLD_KEY = 'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes';
+export const SKEW_FACTOR_KEY = 'spark.sql.adaptive.skewJoin.skewedPartitionFactor';
+export const FORCE_SKEW_JOIN_KEY = 'spark.sql.adaptive.forceOptimizeSkewedJoin';
+const ADVISORY_SIZE_KEY = 'spark.sql.adaptive.advisoryPartitionSizeInBytes';
+
+                                    
+                    
+                                                                                   
+               
+                             
+ 
+
+                                
+                                           
+                                    
+                  
+                                                                                                 
+                                                                           
+                  
+                  
+                                                                                                                  
+                                 
+                                                                                                     
+                                                                                                     
+                                               
+                                      
+                                                                                                    
+                                        
+                                                                                       
+                                          
+                                                                  
+                    
+ 
+
+/** The joins AQE skew-join handling applies to; the name carries `(skew=true)` once it split one. */
+const JOIN_NAME = /^(SortMergeJoin|ShuffledHashJoin)(\(skew=true\))?$/;
+export const isSkewJoinNode = (name        )          => JOIN_NAME.test(name);
+// Operators that wrap a shuffle read without changing what the join reads.
+const WRAPPER_NAME = /^(WholeStageCodegen|InputAdapter$|AQEShuffleRead$|CustomShuffleReader$|ColumnarToRow$)/;
+const SHUFFLE_READ_NAME = /^(AQEShuffleRead|CustomShuffleReader)$/;
+const SPLITTABLE_LEFT = new Set(['Inner', 'Cross', 'LeftSemi', 'LeftAnti', 'LeftOuter']);
+const SPLITTABLE_RIGHT = new Set(['Inner', 'Cross', 'RightOuter']);
+const JOIN_TYPE = /\]\s*,\s*(Inner|Cross|LeftOuter|RightOuter|FullOuter|LeftSemi|LeftAnti|ExistenceJoin)\b/;
+// A partition is only clearly skewed when the largest task read is at least this many times the median.
+const SKEWED_READ_RATIO = 2;
+// Spark judges each join side's partitions against that side's own median. A task's read sums both
+// sides, so it stands for one side only when the other contributes at most this share of the bytes.
+const ISOLATED_SIDE_SHARE = 0.05;
+const EXCHANGE_NAME = /Exchange$/;
+const SHUFFLE_ORIGIN = /\b(ENSURE_REQUIREMENTS|REPARTITION_BY_COL|REPARTITION_BY_NUM|REBALANCE_PARTITIONS_BY_NONE|REBALANCE_PARTITIONS_BY_COL)\b/;
+
+/** What one join input reads: a shuffle Spark added for the join, a shuffle the job asked for, or
+ * some other operator (the join reads from it, not from a shuffle). */
+                
+                                               
+                                                                                  
+                                       
+
+                    
+                 
+                                  
+                        
+ 
+
+function metricValue(node          , name        )                {
+  return node.metrics?.find((m) => m.name === name)?.value ?? null;
+}
+
+// Follows one join input down through wrappers (and the Sort a sort-merge join puts over its
+// shuffle) to the operator that produces it.
+function classifySide(input                      )                                         {
+  const reads             = [];
+  let node = input;
+  let sortSeen = false;
+  while (node != null) {
+    if (SHUFFLE_READ_NAME.test(node.name)) reads.push(node);
+    if (WRAPPER_NAME.test(node.name) || (node.name === 'Sort' && !sortSeen)) {
+      if (node.name === 'Sort') sortSeen = true;
+      node = node.children[0];
+      continue;
+    }
+    break;
+  }
+  const coalesced = reads.some((r) => /\bcoalesced\b/.test(r.detail ?? ''));
+  if (node == null) return { side: { kind: 'operator', name: 'an unknown operator' }, reads };
+  if (node.name === 'ShuffleQueryStage') {
+    const origin = SHUFFLE_ORIGIN.exec(node.children[0]?.detail ?? '')?.[1];
+    if (origin == null || origin === 'ENSURE_REQUIREMENTS') return { side: { kind: 'joinShuffle', coalesced }, reads };
+    return { side: { kind: 'userShuffle', hint: origin.startsWith('REBALANCE') ? 'rebalance' : 'repartition', coalesced }, reads };
+  }
+  return { side: { kind: 'operator', name: node.name }, reads };
+}
+
+function describeOperator(name        )         {
+  if (/Aggregate$/.test(name)) return 'an aggregate';
+  if (/^Window/.test(name)) return 'a window';
+  if (/Scan/.test(name)) return 'a scan';
+  return `a ${name} operator`;
+}
+
+// Whether an operator above the join needs the join's output partitioning, so a split (which
+// breaks it) would cost a shuffle. Stops at the first exchange: the stage boundary above it
+// reshuffles anyway. A partial aggregate asks for no distribution.
+function needsJoinPartitioning(ancestors            )                {
+  for (const a of ancestors) {
+    if (/Exchange$|^ShuffleQueryStage$/.test(a.name)) return null;
+    if (/Aggregate$/.test(a.name) && !/partial_/.test(a.detail ?? '')) return 'an aggregate';
+    if (/^Window/.test(a.name)) return 'a window';
+    if (JOIN_NAME.test(a.name)) return 'another join';
+  }
+  return null;
+}
+
+function collectJoins(root          )             {
+  const parents = new Map                           ();
+  const joins             = [];
+  walkPlanTree(root, (node, parent) => {
+    parents.set(node, parent);
+    if (JOIN_NAME.test(node.name)) joins.push(node);
+  });
+  return joins.map((node) => {
+    const ancestors             = [];
+    for (let p = parents.get(node) ?? null; p != null; p = parents.get(p) ?? null) ancestors.push(p);
+    return { node, ancestors };
+  });
+}
+
+/** Whether the execution's final plan shows AQE splitting a skewed partition (an `AQEShuffleRead
+ * skewed` node), which adds tasks beyond the configured partition count. Null when the plan is not a
+ * final adaptive plan, so the log cannot say. */
+export function planShowsSkewSplit(plan                             )                 {
+  if (plan == null || plan.name !== 'AdaptiveSparkPlan' || !/isFinalPlan=true/.test(plan.detail ?? '')) return null;
+  let split = false;
+  walkPlanTree(plan, (node) => { if (SHUFFLE_READ_NAME.test(node.name) && /\bskewed\b/.test(node.detail ?? '')) split = true; });
+  return split;
+}
+
+// The bytes one join input shuffled, from its exchange's metrics (null when the plan has none).
+function sideShuffleBytes(input                      , metrics                    = ['shuffle bytes written', 'data size'])                {
+  let bytes                = null;
+  walkPlanTree(input, (n) => {
+    if (bytes != null || !EXCHANGE_NAME.test(n.name)) return;
+    for (const name of metrics) {
+      const value = metricValue(n, name);
+      if (value != null) { bytes = value; return; }
+    }
+  });
+  return bytes;
+}
+
+// Whether the stage reads this join's shuffles: an exchange of the join lists the stage, else the
+// stage's shuffle read matches the bytes the join's two exchanges wrote (a stage reads exactly what
+// the shuffles feeding it wrote). False when the plan gives neither, which no join can be assumed from.
+function stageReadsJoin(join          , input               )          {
+  const exchanges             = [];
+  walkPlanTree(join.node, (n) => { if (EXCHANGE_NAME.test(n.name)) exchanges.push(n); });
+  const withStages = exchanges.filter((n) => (n.stageIds?.length ?? 0) > 0);
+  if (withStages.length > 0) return withStages.some((n) => n.stageIds .includes(input.stageId));
+  const left = sideShuffleBytes(join.node.children[0], ['shuffle bytes written']);
+  const right = sideShuffleBytes(join.node.children[1], ['shuffle bytes written']);
+  const read = input.stageShuffleReadBytes ?? 0;
+  if (left == null || right == null || !(read > 0) || !(left + right > 0)) return false;
+  return Math.abs(read - (left + right)) <= 0.1 * (left + right);
+}
+
+function sideNoun(index       )                   { return index === 0 ? 'left' : 'right'; }
+
+function diagnoseJoin(join          , input               )                           {
+  const { node } = join;
+  const left = classifySide(node.children[0]);
+  const right = classifySide(node.children[1]);
+  const reads = [...left.reads, ...right.reads];
+
+  // Split: the join carries `skew=true` and a read on one side says `skewed`.
+  const splitReads = reads.filter((r) => /\bskewed\b/.test(r.detail ?? ''));
+  if (/\(skew=true\)$/.test(node.name) || splitReads.length > 0) {
+    const partitions = splitReads.reduce((sum, r) => sum + (metricValue(r, 'number of skewed partitions') ?? 0), 0);
+    const splits = splitReads.reduce((sum, r) => sum + (metricValue(r, 'number of skewed partition splits') ?? 0), 0);
+    const what = partitions > 0 && splits > 0
+      ? `AQE already split ${partitions} skewed partition${partitions === 1 ? '' : 's'} into ${splits} tasks`
+      : 'AQE already split the skewed partitions';
+    // A tail the data volume explains is not a GC pause or a slow host, so the split did not cure it.
+    if (input.tailData != null) {
+      const ratio = input.tailData.ratio != null ? ` (a median ${Math.round(input.tailData.ratio * 100) / 100}× the data of the median task)` : '';
+      return {
+        case: 'split',
+        text: `${what}, yet the slow tasks still read far more data than the median task${ratio}, so skew remains that the split did not remove: ${input.keyRemedy}`,
+        remediation: [codeFix(input.keyRemedy)],
+      };
+    }
+    return {
+      case: 'split',
+      text: `${what}, so the imbalance that remains is not join skew: look for a GC pause, a slow host or an expensive key instead`,
+      remediation: [],
+    };
+  }
+
+  // Even reads: with the largest task read close to the median, no partition stands out for AQE to
+  // split. A coalesced read is a sum of partitions, so its ratio says nothing about one partition.
+  const coalesced = (left.side.kind !== 'operator' && left.side.coalesced) || (right.side.kind !== 'operator' && right.side.coalesced);
+  const ratio = input.readP50 > 0 ? Math.round(input.readMax / input.readP50 * 100) / 100 : null;
+  // Spark compares a partition with its own side's median, so a ratio of summed reads only speaks
+  // for a side when the other side adds next to nothing. It also would not see the data skew a tail
+  // attributed to data volume reports.
+  const leftBytes = sideShuffleBytes(node.children[0]);
+  const rightBytes = sideShuffleBytes(node.children[1]);
+  const sideTotal = (leftBytes ?? 0) + (rightBytes ?? 0);
+  const isolatedSide = leftBytes != null && rightBytes != null && sideTotal > 0
+    && Math.min(leftBytes, rightBytes) <= ISOLATED_SIDE_SHARE * sideTotal;
+  if (!coalesced && isolatedSide && input.tailData == null && input.readP50 > 0 && input.readMax < SKEWED_READ_RATIO * input.readP50) {
+    return {
+      case: 'evenReads',
+      text: `its shuffle reads are even (the largest task reads ${formatBytes(input.readMax)}, ${ratio}× the median), so AQE has no skewed partition to split and the slow tail is not partition-size skew: look for a GC pause, a slow host or an expensive key instead`,
+      remediation: [],
+    };
+  }
+
+  // Plan shape: both inputs must be shuffles Spark added for this join.
+  const blocked = [left.side, right.side].map((s, i) => ({ s, i: i          })).find((x) => x.s.kind === 'operator');
+  if (blocked?.s.kind === 'operator') {
+    const above = describeOperator(blocked.s.name);
+    return {
+      case: 'planShape',
+      text: `AQE skew-join handling cannot apply: the join's ${sideNoun(blocked.i)} input comes from ${above} instead of a shuffle, and AQE splits only a shuffle that feeds the join directly; ${input.keyRemedy}`,
+      remediation: [codeFix(input.keyRemedy)],
+    };
+  }
+  const user = [left.side, right.side].find((s) => s.kind === 'userShuffle');
+  if (user?.kind === 'userShuffle') {
+    return {
+      case: 'userRepartition',
+      text: `a ${user.hint} you wrote feeds the join, and AQE leaves a shuffle you asked for as it is (it splits only shuffles Spark added for the join): drop the ${user.hint} so the join's own shuffle can be split, or ${input.keyRemedy}`,
+      remediation: [codeFix(`drop the explicit ${user.hint} ahead of the join, or ${input.keyRemedy}`)],
+    };
+  }
+
+  // Join type: which sides AQE may split.
+  const joinType = JOIN_TYPE.exec(node.detail ?? '')?.[1];
+  const canLeft = joinType != null && SPLITTABLE_LEFT.has(joinType);
+  const canRight = joinType != null && SPLITTABLE_RIGHT.has(joinType);
+  if (joinType != null && !canLeft && !canRight) {
+    return {
+      case: 'joinType',
+      text: `AQE never splits either side of a ${joinType} join: join the hot key on its own and union the results, or ${input.keyRemedy}`,
+      remediation: [codeFix(`join the hot key separately and union the results, or ${input.keyRemedy}`)],
+    };
+  }
+
+  // Thresholds. A task reads one partition of each side (more when AQE coalesced them), so the
+  // task's read is an upper bound on any one partition: a task under the threshold proves no
+  // partition is over it. The factor and a coalesced stage's reads are only comparable when the
+  // read is not a sum of coalesced partitions.
+  const threshold = parseSparkBytes(input.conf(SKEW_THRESHOLD_KEY));
+  const factor = Number.parseFloat(input.conf(SKEW_FACTOR_KEY) ?? '');
+  const underThreshold = threshold != null && input.readMax <= threshold;
+  const underFactor = !coalesced && isolatedSide && Number.isFinite(factor) && input.readP50 > 0 && input.readMax <= factor * input.readP50;
+  const biggest = `its largest shuffle partition (${formatBytes(input.readMax)})`;
+  if (underThreshold && underFactor) {
+    return {
+      case: 'belowThreshold',
+      text: `${biggest} is under the ${input.conf(SKEW_THRESHOLD_KEY)} threshold (${SKEW_THRESHOLD_KEY}) and only ${ratio}× the median, under the ${factor}× factor (${SKEW_FACTOR_KEY}) AQE needs before it treats a partition as skewed, so lower both for this query`,
+      remediation: [decreaseConf(SKEW_THRESHOLD_KEY), decreaseConf(SKEW_FACTOR_KEY)],
+    };
+  }
+  if (underThreshold) {
+    return {
+      case: 'belowThreshold',
+      text: `${biggest} is under the ${input.conf(SKEW_THRESHOLD_KEY)} AQE needs before it treats a partition as skewed (${SKEW_THRESHOLD_KEY}), so lower the threshold for this query`,
+      remediation: [decreaseConf(SKEW_THRESHOLD_KEY)],
+    };
+  }
+  if (underFactor) {
+    return {
+      case: 'belowThreshold',
+      text: `${biggest} is only ${ratio}× the median, under the ${factor}× AQE needs before it treats a partition as skewed (${SKEW_FACTOR_KEY}), so lower the factor for this query`,
+      remediation: [decreaseConf(SKEW_FACTOR_KEY)],
+    };
+  }
+  const advisory = parseSparkBytes(input.conf(ADVISORY_SIZE_KEY));
+  // Coalescing merges small partitions up to the advisory size on each side, so a task read below
+  // twice that can be many partitions that are each under the threshold.
+  if (coalesced && advisory != null && input.readMax <= 2 * advisory) return null;
+
+  const needing = needsJoinPartitioning(join.ancestors);
+  // forceOptimizeSkewedJoin exists from Spark 3.3 (no default before), so earlier runs get the key
+  // remedy even when they log the property. A run with no recorded version can force when it logs it.
+  const force = input.conf(FORCE_SKEW_JOIN_KEY);
+  const versionKnown = /^\d+\.\d+/.test(input.sparkVersion ?? '');
+  const canForce = versionKnown ? sparkConfDefault(input.sparkVersion, FORCE_SKEW_JOIN_KEY) != null : force != null;
+  const forced = canForce && force?.toLowerCase() === 'true';
+  const extraShuffleText = needing == null || forced ? null
+    : `AQE skipped splitting it because ${needing} above the join needs the join's partitioning and a split would add a shuffle: ${canForce ? `set ${FORCE_SKEW_JOIN_KEY}=true if that shuffle costs less than the tail, or ` : ''}${input.keyRemedy}`;
+
+  if (joinType != null && canLeft !== canRight) {
+    const only = canLeft ? 'left' : 'right';
+    const other = canLeft ? 'right' : 'left';
+    const tail = needing == null || forced ? ''
+      : `; if the skew is on the ${only} side, AQE skipped it because ${needing} above the join needs the join's partitioning and a split would add a shuffle${canForce ? ` (${FORCE_SKEW_JOIN_KEY}=true accepts that)` : ''}`;
+    // The other side supplies the nulls or only filters, and writing the join the other way round
+    // keeps it that side's table (or changes the join's meaning), so swapping sides cannot help.
+    const hotKey = `join the hot key on its own and union the results, or ${input.keyRemedy}`;
+    return {
+      case: 'joinType',
+      text: `AQE can split only the ${only} side of a ${joinType} join, so a skewed partition on the ${other} side stays whole: if that is where the skew is, ${hotKey}${tail}`,
+      remediation: [codeFix(hotKey), ...(tail && canForce ? [setConf(FORCE_SKEW_JOIN_KEY, true)] : [])],
+    };
+  }
+  if (extraShuffleText != null) {
+    return { case: 'extraShuffle', text: extraShuffleText, remediation: [canForce ? setConf(FORCE_SKEW_JOIN_KEY, true) : codeFix(input.keyRemedy)] };
+  }
+  return {
+    case: 'notSplit',
+    text: `AQE did not split it although the partition is over the thresholds and the plan allows it, and the log does not say why (a partition built from a single map task cannot be split, for one): ${input.keyRemedy}`,
+    remediation: [codeFix(input.keyRemedy)],
+  };
+}
+
+/** Which of Spark's skew-join conditions applies to the join running in `stageId`, or null when
+ * the plan cannot say: it is not a final adaptive plan, no join can be tied to the stage, or the
+ * stage's read is inconclusive (a coalesced read that may be a sum of small partitions). */
+export function diagnoseJoinSkew(input               )                           {
+  const { plan } = input;
+  if (plan == null || plan.name !== 'AdaptiveSparkPlan' || !/isFinalPlan=true/.test(plan.detail ?? '')) return null;
+  const joins = collectJoins(plan);
+  let chosen = joins.filter((j) => j.node.stageIds?.includes(input.stageId));
+  // A plan whose joins carry no stage ids at all can still be read when it has a single join.
+  // A join with no stage ids stands for the stage only when the stage reads its shuffles.
+  if (chosen.length === 0 && joins.length === 1 && !joins[0].node.stageIds?.length && stageReadsJoin(joins[0], input)) chosen = joins;
+  const diagnoses = chosen.map((j) => diagnoseJoin(j, input)).filter((d)                         => d != null);
+  return diagnoses.find((d) => d.case === 'split') ?? diagnoses[0] ?? null;
+}

package/vendor-core/cli/budgets.js

@@ -2,7 +2,7 @@ import { computeEfficiencyModel } from '../efficiency-model.js';
 import { computeSkewRatio, ENTRY_BY_TYPE,                         } from '../detectors.js';
 import { effectiveThresholds } from '../threshold-overrides.js';
 import { IMPACT_BAND_ORDER } from '../format-utils.js';
-                                                              
+import { isEscalation,                        } from '../run-comparison.js';
 import { PAIR_DELTA_METRICS,                                      } from '../stage-alignment.js';
                                                                  
                                                                 
@@ -205,11 +205,13 @@ function checkFailOnIntroduced(comparison                   , band        )
   if (band !== 'all' && !IMPACT_BANDS.includes(band              )) {
     return { name: 'fail-on-introduced', status: 'inconclusive', detail: `Impact band "${band}" is not recognized (expected "all" or one of ${IMPACT_BANDS.join(', ')}).` };
   }
-  const matches = band === 'all'
+  const introduced = band === 'all'
     ? comparison.findings.introduced
     : comparison.findings.introduced.filter((f) => f.impactBand === band);
-  return matches.length > 0
-    ? { name: 'fail-on-introduced', status: 'violation', detail: `${matches.length} introduced finding(s) match "${band}".` }
+  const escalated = band === 'all' ? [] : comparison.findings.changed.filter((c) => isEscalation(c) && c.candImpactBand === band);
+  const count = introduced.reduce((n, f) => n + f.candCount, 0) + escalated.reduce((n, c) => n + c.count, 0);
+  return count > 0
+    ? { name: 'fail-on-introduced', status: 'violation', detail: `${count} introduced finding(s) match "${band}".` }
     : { name: 'fail-on-introduced', status: 'pass', detail: `No introduced findings match "${band}".` };
 }
 

package/vendor-core/cli/collect-run-pair.js

@@ -0,0 +1,89 @@
+import { readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { Worker } from 'node:worker_threads';
+import { collectRun } from './collect-run.js';
+import { mcpError } from '../mcp-error.js';
+
+                                                           
+
+// Parsing is single-threaded JS, so two logs parsed one after the other take the sum of their
+// times, and parsing both on one thread (interleaved) takes the same. A worker thread parses one of
+// them in parallel at the price of a thread start and a structured clone of the model, so it pays
+// only when the log it parses is big enough (worker-protocol.md, "Decoder and parser calibration").
+const COMPRESSED_LOG = /\.(zstd?|lz4|snappy|zip|gz)$/i;
+// A compressed log holds about this many times its size in event text, and parses at about that
+// many times the cost per byte on disk.
+const COMPRESSED_EXPANSION = 8;
+/** The smaller log's estimated event text (bytes) below which both logs parse in this thread. */
+export const MIN_OFFLOAD_PARSE_BYTES = 32 * 1024 * 1024;
+
+/** Event-text bytes a log is expected to parse, from its size on disk. 0 when it cannot be read:
+ * `collectRun` reports that failure itself. */
+export function estimatedParseBytes(path        )         {
+  try {
+    const stat = statSync(path);
+    if (stat.isDirectory()) return readdirSync(path).reduce((sum, name) => sum + estimatedParseBytes(join(path, name)), 0);
+    return COMPRESSED_LOG.test(path) ? stat.size * COMPRESSED_EXPANSION : stat.size;
+  } catch {
+    return 0;
+  }
+}
+
+/** `collectRun` on a worker thread. A worker that cannot start, dies (out of memory included) or
+ * returns a model it cannot clone falls back to parsing in this thread; a log the parser rejects
+ * rejects here with the same message and code. */
+export function collectRunInWorker(path        )                        {
+  return new Promise((resolve, reject) => {
+    let done = false;
+    let worker        ;
+    try {
+      worker = new Worker(new URL('./collect-run-worker.js', import.meta.url), { workerData: { path } });
+    } catch {
+      // The thread could not be created at all (Node's permission mode without worker access).
+      collectRun(path).then(resolve, reject);
+      return;
+    }
+    const finish = (settle            )       => {
+      if (done) return;
+      done = true;
+      void worker.terminate();
+      settle();
+    };
+    const parseHere = ()       => finish(() => { collectRun(path).then(resolve, reject); });
+    worker.once('message', (msg                                                                     ) => {
+      if (msg.run) finish(() => resolve(msg.run ));
+      else finish(() => reject(mcpError(msg.failed?.code ?? 'invalid-event-log', msg.failed?.message ?? 'The event log could not be parsed.')));
+    });
+    worker.once('error', parseHere);
+    worker.once('exit', parseHere);
+  });
+}
+
+                                
+                                               
+                                                                          
+                                                 
+ 
+
+const settle =    (promise            )                                   =>
+  promise.then((value) => ({ status: 'fulfilled', value }), (reason) => ({ status: 'rejected', reason }));
+
+/** Collects a baseline and a candidate run. When even the smaller log is big enough to repay a
+ * thread, it parses in a worker while this thread parses the other, so the pair takes about as long
+ * as the bigger log alone; otherwise the two parse here, baseline first, and a baseline that fails
+ * stops the candidate from being read. The model is the same either way. */
+export async function collectRunPair(
+  baselinePath        , candidatePath        , { minOffloadBytes = MIN_OFFLOAD_PARSE_BYTES } = {},
+)                         {
+  const baselineBytes = estimatedParseBytes(baselinePath), candidateBytes = estimatedParseBytes(candidatePath);
+  if (Math.min(baselineBytes, candidateBytes) < minOffloadBytes) {
+    const baseline = await settle(collectRun(baselinePath));
+    return baseline.status === 'rejected' ? { baseline } : { baseline, candidate: await settle(collectRun(candidatePath)) };
+  }
+  const offloadBaseline = baselineBytes <= candidateBytes;
+  const [baseline, candidate] = await Promise.all([
+    settle(offloadBaseline ? collectRunInWorker(baselinePath) : collectRun(baselinePath)),
+    settle(offloadBaseline ? collectRun(candidatePath) : collectRunInWorker(candidatePath)),
+  ]);
+  return { baseline, candidate };
+}

package/vendor-core/cli/collect-run-worker.js

@@ -0,0 +1,16 @@
+// Worker-thread entry of collect-run-pair.ts: parses one log and posts the finished model back.
+// Plain JS, like the other worker entries, so a published install (which ships pre-stripped .js)
+// and the monorepo (which runs the .ts directly) load it the same way.
+import { parentPort, workerData } from 'node:worker_threads';
+import { collectRun } from './collect-run.js';
+
+let run;
+try {
+  run = await collectRun(workerData.path);
+} catch (e) {
+  // A log that cannot be parsed fails the same way in the parent, so it is reported, not retried.
+  parentPort.postMessage({ failed: { message: e?.message ?? String(e), code: e?.code } });
+}
+// A model the thread cannot clone, or a worker that dies, reaches the parent as a worker error,
+// which parses the log in the parent's own thread instead.
+if (run) parentPort.postMessage({ run });

package/vendor-core/cli/collect-run.js

@@ -7,8 +7,7 @@ import { routeMessage,                     } from '../ingest.js';
                                             
 import { mcpError } from '../mcp-error.js';
 
-// No whole-file arrayBuffer(): the parser only ever reads bounded slices, and a
-// whole-file read is what capped local logs at 2 GiB.
+// No whole-file arrayBuffer(): the parser reads bounded slices, so a local log is not capped at 2 GiB.
                            
                              
                                                                              

package/vendor-core/cli/native-zstd.js

@@ -1,11 +1,11 @@
 // Node-only zstd decoding for the CLI/MCP parse path: Node's native zlib zstd, one frame at a
 // time, with the vendored fzstd (what the browser uses) as the fallback.
 //
-// Spark writes an event log as thousands of small zstd frames (one per flush: 10217 frames for
-// the largest real log's 3.5 GB). Node's zstdDecompressSync and createZstdDecompress both stop
-// after the first frame (the stream then fails with "Unknown frame descriptor"), so this walks
-// frame boundaries itself and decompresses each complete frame natively: 3-5x faster than fzstd
-// on the real logs (5.6s -> 1.3-1.8s on the largest).
+// Spark writes an event log as thousands of small zstd frames (one per flush). Node's
+// zstdDecompressSync and createZstdDecompress both stop after the first frame (the stream then
+// fails with "Unknown frame descriptor"), so this walks frame boundaries itself and decompresses
+// each complete frame natively, 3-5x faster than fzstd (worker-protocol.md, "Decoder and parser
+// calibration").
 import * as zlib from 'node:zlib';
 import { Decompress as ZstdDecompress } from '../vendor/fzstd.js';
 
@@ -102,8 +102,7 @@ export const nativeZstdAvailable =
 // falls back to fzstd when frameEnd says NOT_NATIVE, when it is still incomplete after
 // `maxFrameBytes` compressed bytes, or when a final push ends inside it. A frame that arrives
 // over many pushes is gathered into one buffer that grows by doubling, and its walk resumes where
-// the last push left it: copying the gathered bytes into a new buffer and walking the frame from
-// its start on every push cost about 4 GB of copying near the 64 MB limit.
+// the last push left it, so a large frame is not copied and re-walked on every push.
 class FrameSplitter {
   // The incomplete frame's bytes so far, gathered[0, gatheredLength); null when none is pending.
           gathered                    = null;
@@ -188,7 +187,7 @@ function decompressBounded(frame            , maxBytes        )
 
 // Same push(chunk, final) contract as fzstd's Decompress. Once a frame falls back, fzstd takes
 // the rest of the stream from that frame's first byte; a truncated last frame goes to fzstd too,
-// so it fails with fzstd's own "unexpected EOF", as it always has. `maxFrameBytes` is for tests.
+// so it fails with fzstd's own "unexpected EOF". `maxFrameBytes` is for tests.
 export function createNativeZstdDecoder(
   onChunk                             ,
   { maxFrameBytes = MAX_NATIVE_FRAME_BYTES }                             = {},
@@ -218,15 +217,13 @@ export function createNativeZstdDecoder(
 
 // Frames at least this big (compressed) are decompressed on libuv's threadpool, up to
 // MAX_FRAMES_IN_FLIGHT at a time, while the main thread parses the output of earlier ones.
-// Smaller frames cost less to decompress inline than to hand off (p50 is 5.8 KB of output on the
-// largest real log; 16-64 KB thresholds measured the same, 256 KB lost most of the gain). On that
-// log 4 in flight (the pool's default size) parsed in 3.1s at 711 MB peak RSS, 8 in 3.2s at 886 MB
-// and 2 in 3.5s at 607 MB. The pool's output chunk size barely mattered (64 KB to 1 MB).
+// Smaller frames cost less to decompress inline than to hand off. 4 in flight is the threadpool's
+// default size. Measurements: worker-protocol.md, "Decoder and parser calibration".
 const THREADED_MIN_FRAME_BYTES = 64 * 1024;
 const MAX_FRAMES_IN_FLIGHT = 4;
 const THREADED_CHUNK_BYTES = 256 * 1024;
-// Small frames that arrive while an earlier frame is in flight wait in the queue behind it. With
-// no cap, a first version held most of the largest real log's output (RSS 0.6 -> 4.1 GB).
+// Small frames that arrive while an earlier frame is in flight wait in the queue behind it; the
+// cap bounds the output they hold.
 const MAX_QUEUED_FRAMES = 64;
 
 // One frame decompressing on the threadpool. Node's zstd stream ends after one frame, so each
@@ -283,8 +280,7 @@ class OffThreadFrame {
 // push must be awaited before the next. It keeps references to pushed bytes until their frames
 // are decoded, so callers must not reuse a pushed buffer. Fallback to fzstd, for the same frames
 // as createNativeZstdDecoder, happens only after every queued frame is delivered; a failed frame
-// rejects the push that reaches it. `threadedMinFrameBytes` is for tests. Measured on the 14 real
-// logs: parse 9.4s -> 8.1s; logs with few large frames (12 of 6455 on a 28 MB one) gain nothing.
+// rejects the push that reaches it. `threadedMinFrameBytes` is for tests.
 export function createThreadedZstdDecoder(
   onChunk                             ,
   { maxFrameBytes = MAX_NATIVE_FRAME_BYTES, threadedMinFrameBytes = THREADED_MIN_FRAME_BYTES }