sparkforensics-cli 0.1.0 → 0.2.2

package/vendor-core/redact.js

@@ -7,6 +7,8 @@
 // Deterministic (sorted assignment), idempotent (pseudonyms map to themselves),
 // and non-mutating (returns a fresh, deep-copied tree).
 
+                                                      
+
 // Host / IP identifier patterns. Used to enumerate host names that surface only
 // inside free text: recommendation strings, `stageFailed`'s failure-reason
 // value, SQL relation/node names, never as a structured `host` field, so
@@ -23,7 +25,7 @@ const HOST_PATTERNS = [
 // Spark application id (e.g. application_1690000000000_0001). Besides
 // redactReport's structured summary.app.id field, an app id can also surface
 // as a residual free-text token (e.g. embedded in a finding's evidence or
-// recommendation string) — same as a host/IP token embedded in a stage name
+// recommendation string), same as a host/IP token embedded in a stage name
 // or plan detail. redactComparison has no structured app-id field at all
 // (baselineLabel/candidateLabel are caller-supplied labels, not Spark app
 // ids), so free text is its *only* source of app ids.
@@ -57,19 +59,58 @@ function scanHostTokens(node         , hosts             )       {
   scanTokens(node, [{ patterns: HOST_PATTERNS, out: hosts }]);
 }
 
+// Recursively collects every string value found under a key literally named
+// `host`, anywhere in the tree. Host names surface at several depths, a
+// finding's own `host`, `evidence.host`, and now
+// `evidence.failedTaskDetails[].host` / `evidence.retriedTaskDetails[].host`,
+// and only some of them are host-*pattern*-shaped (a plain FQDN like
+// `worker-3.internal` matches neither HOST_PATTERN). Walking by key name rather
+// than enumerating known paths keeps any future nested `host` field covered.
+function collectHostFields(node         , hosts             )       {
+  if (Array.isArray(node)) {
+    for (const n of node) collectHostFields(n, hosts);
+    return;
+  }
+  if (node && typeof node === 'object') {
+    for (const [k, v] of Object.entries(node)) {
+      if (k === 'host' && typeof v === 'string' && v.length > 0) hosts.add(v);
+      else collectHostFields(v, hosts);
+    }
+  }
+}
+
+// Spark config keys ending in `host`/`hostname` (e.g. spark.driver.host,
+// spark.yarn.am.hostname) carry plain FQDN host names that neither
+// HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
+// walk catches (the literal key is the dotted Spark property name, never
+// `host` itself). app.config is a flat Record<string, string> unique to
+// redactExportData: no other redact* export ships a raw Spark config dict.
+// Known gap: a hostname value under a differently-named key isn't caught by
+// this suffix check. Confirmed against a real cluster config: spark.master,
+// spark.yarn.historyServer.address, and the plural YARN proxy/HA keys
+// (...AmIpFilter.param.PROXY_HOSTS, ...RM_HA_URLS) all carry real hostnames
+// through un-redacted today.
+function collectConfigHostValues(config                                    , hosts             )       {
+  if (!config) return;
+  for (const [key, value] of Object.entries(config)) {
+    const lastSegment = key.slice(key.lastIndexOf('.') + 1).toLowerCase();
+    if ((lastSegment === 'host' || lastSegment === 'hostname') && value) hosts.add(value);
+  }
+}
+
 // Known locations of the identifiers, so we don't have to guess which strings
-// are sensitive: the app id lives at summary.app.id; host names live on each
-// finding's `host` field. The real report nests that host under `evidence.host`
-// (host is not a first-class report column), so read both shapes, plus a
-// free-text scan for hosts/IPs and app ids that appear only inside string
-// values (e.g. a finding's evidence/recommendation text).
+// are sensitive: the app id lives at summary.app.id; host names live under any
+// `host` key in the findings tree (see collectHostFields), plus a free-text
+// scan for hosts/IPs and app ids that appear only inside string values (e.g. a
+// finding's evidence/recommendation text).
                             
-                                                                     
-                                                                        
-                                                                      
-                                                                           
+                                                                          
+                                                                              
                                              
-                                                                    
+                                                                               
+                                                                              
+                 
+                       
  
 
 function collectIds(report                  )                                              {
@@ -77,11 +118,7 @@ function collectIds(report                  )
   const hosts = new Set        ();
   const appId = report?.summary?.app?.id;
   if (typeof appId === 'string' && appId.length > 0) appIds.add(appId);
-  for (const f of report?.findings ?? []) {
-    for (const h of [f?.host, f?.evidence?.host]) {
-      if (typeof h === 'string' && h.length > 0) hosts.add(h);
-    }
-  }
+  collectHostFields(report?.findings, hosts);
   // Free-text scan for both host/IP tokens and app-id tokens: an app id can
   // surface in a finding's evidence/recommendation text (e.g. "retry app
   // application_1690000000000_0001 failed") same as redactComparison's scan.
@@ -173,3 +210,22 @@ export function redactComparison   (comparison   )    {
   scanTokens(comparison, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
   return applyReplacements(comparison, { appIds, hosts });
 }
+
+// HTML-export counterpart to redactReport/redactComparison. Unlike
+// redactReport (scoped to EvidenceReportJson's summary.app.id + findings),
+// this also walks executors.added/removed for their literal `host` field
+// (ExecutorAddedEvent.host), since raw executor records: not just findings
+//: reach data.js.
+export function redactExportData(data               )                {
+  const appIds = new Set        ();
+  const hosts = new Set        ();
+  const appId = data.app?.id;
+  if (typeof appId === 'string' && appId.length > 0) appIds.add(appId);
+  collectHostFields(data.executors.added, hosts);
+  collectHostFields(data.executors.removed, hosts);
+  collectHostFields(data.catalog, hosts);
+  collectHostFields(data.configFindings, hosts);
+  collectConfigHostValues(data.app?.config, hosts);
+  scanTokens(data, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
+  return applyReplacements(data, { appIds, hosts });
+}

package/vendor-core/rolling-log-reassembly.js

@@ -1,9 +1,7 @@
-// Rolling event-log directory/zip member-name reassembly. Pulled out of
-// shs-fetch.ts (Task: bundle-size fix) so DropZone.tsx's drag-drop path can
-// import just this reassembly logic without pulling in the Zod schemas and
-// vendored zstd/gzip decompressors that the rest of shs-fetch.ts (and the
-// wider parser-worker.ts module graph) depend on. Keep this file free of any
-// import beyond plain JS/TS built-ins.
+// Rolling event-log directory/zip member-name reassembly. Import-free (plain
+// JS/TS built-ins only) so DropZone.tsx's drag-drop path can use it without
+// pulling in the Zod schemas and vendored zstd/gzip decompressors that the rest
+// of shs-fetch.ts depends on. Keep it that way.
 
 export function naturalCompare(a        , b        )         {
   const tokenize = (s        ) => s.match(/(\d+)|(\D+)/g) ?? [s];

package/vendor-core/run-comparison.js

@@ -1,4 +1,3 @@
-// src/run-comparison.ts
 import { computeWallClock } from './wall-clock.js';
 import { normalizeDetail } from './detectors.js';
 import { captureSnapshot } from './session-snapshot.js';
@@ -20,7 +19,7 @@ import { captureSnapshot } from './session-snapshot.js';
                           
                             
                                                                              
-                                                                                                                  
+                                                                                                                                                  
                                                                             
                                                                             
  
@@ -36,16 +35,9 @@ export function normalizeStageName(name        )         {
     .trim();
 }
 
-// cyrb53: fast, non-cryptographic 53-bit string hash, deterministic, portable
-// (runs in-browser worker and Node CLI, no crypto module dependency), fixed-
-// length hex output regardless of input size. src/analyzer.js already has a
-// small stable-hash helper (`fnv1a`, 32-bit) for deterministic finding ids,
-// but that runs once per finding; planTreeIdentity's `visit` below runs once
-// per PLAN NODE, for every stage, in both runs of a comparison: orders of
-// magnitude more hash calls, so the smaller 32-bit space's birthday-collision
-// risk is a real concern here in a way it isn't for finding ids. 53 bits keeps
-// collision probability negligible at that scale; adequate for a same-run
-// identity key, not a security boundary.
+// cyrb53: fast, non-cryptographic, deterministic 53-bit string hash. Not a
+// security boundary; 53 bits keeps collision risk negligible at the per-node,
+// per-stage call volume planTreeIdentity/sqlNodeIdentity put through it.
 function cyrb53(str        , seed = 0)         {
   let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed;
   for (let i = 0; i < str.length; i++) {
@@ -59,38 +51,16 @@ function cyrb53(str        , seed = 0)         {
 }
 
 // Bottom-up, order-independent structural identity of a resolved plan tree:
-// each node folds its own normalized name and normalized `detail` with its
-// already-computed children's DIGESTS (children sorted, so sibling
-// reordering across runs (e.g. AQE picking a different broadcast side)
-// still matches), so two plans collide only when their whole shape AND every
-// node's normalized detail agree. `normalizeDetail` (src/detectors.js, built
-// for cachingOpportunity's composite join/union detection) strips expr ids,
-// `plan_id=`, codegen-stage numbers, and AQE's BuildLeft/BuildRight choice,
-// and canonicalizes commutative equality operands: exactly the run-to-run
-// noise a cross-run identity needs to ignore. Replaces the previous
-// flatten-and-sort-names identity, which ignored `detail` entirely (so a
-// join on one column and a join on another, with identical operator names,
-// collided) and ignored nesting (so two differently-shaped trees sharing a
-// node-name multiset also collided).
-//
-// Each node is encoded via JSON.stringify of its [name, detail, childDigests]
-// triple rather than an ad hoc delimiter-joined string: real Spark `detail`
-// text routinely contains its own `<`/`>`/`{`/`}`/`,` characters (e.g.
-// `ReadSchema: struct<id:bigint>`), so a hand-rolled delimiter scheme could
-// let two structurally different plans serialize to the same string;
-// JSON.stringify escapes/quotes each component so the encoding is
-// unambiguous. Critically, that JSON payload is then folded down to a fixed-
-// length `cyrb53` digest before being returned: the payload only ever
-// contains a node's own (short) name/detail plus an array of already-hashed,
-// fixed-length child digests, never a raw nested JSON blob. Embedding full
-// child identity STRINGS instead (this function's first version) made each
-// level re-stringify-and-escape the level below, so identity length grew
-// ~2^depth; real Spark plans routinely nest 20-60+ operators deep
-// (Scan→Filter→Project→HashAggregate→Exchange→Sort→Join, repeated per
-// join/aggregate), which blew up to megabytes-per-stage or a
-// `RangeError: Invalid string length` well within that range. Digest-folding
-// keeps per-node output size bounded by name+detail length plus a small
-// constant per child, independent of subtree depth.
+// each node folds its normalized name/detail with its children's digests
+// (children sorted, so AQE picking a different broadcast side still matches),
+// so two plans collide only when their whole shape and every node's detail
+// agree. `normalizeDetail` strips the run-to-run noise (expr ids, `plan_id=`,
+// codegen numbers, AQE build-side choice, commutative-operand order). Each
+// node folds to a fixed-length cyrb53 digest (JSON.stringify-encoded, so
+// detail text containing `<`/`>`/`{`/`,` can't collide two different plans)
+// instead of embedding full child identity strings, which would re-escape
+// every level below and blow identity size up to ~2^depth on the 20-60+
+// operator-deep plans real Spark produces.
 function planTreeIdentity(root                             )                {
   if (!root) return null;
   function visit(node          )         {
@@ -100,12 +70,27 @@ function planTreeIdentity(root                             )                {
   return visit(root);
 }
 
-// Plan identity for the stage's SQL execution, so stage identity also requires
-// the SQL-node identity to agree. Empty string when no SQL/plan.
+// Plan identity for the stage's SQL execution, scoped to only the plan nodes
+// this stage actually ran (`node.stageIds`), not the whole tree: two stages
+// sharing one SQL execution (e.g. a self-join's two Exchange stages) otherwise
+// collapse onto one identity regardless of which part of the plan each ran.
+// Falls back to the coarser whole-tree identity when the stage has no
+// attributed nodes (hand-built snapshots without `stageIds`, or unmatched
+// accumulables).
 function sqlNodeIdentity(stage       , snapshot                 )         {
   const execId = stage.sqlExecutionId;
   if (execId == null) return '';
-  return planTreeIdentity(snapshot.sql.get(execId)?.planTree ?? null) ?? '';
+  const root = snapshot.sql.get(execId)?.planTree ?? null;
+  if (!root) return '';
+  const fingerprints           = [];
+  (function collect(node          )       {
+    if (node.stageIds?.includes(stage.id)) {
+      fingerprints.push(JSON.stringify([normalizeStageName(node.name ?? ''), normalizeDetail(node.detail ?? '')]));
+    }
+    for (const child of node.children ?? []) collect(child);
+  })(root);
+  if (fingerprints.length === 0) return planTreeIdentity(root) ?? '';
+  return cyrb53(JSON.stringify(fingerprints.sort()));
 }
 
 export function stageIdentity(stage       , snapshot                 )         {
@@ -113,7 +98,7 @@ export function stageIdentity(stage       , snapshot                 )         {
 }
 
 function identityIndex(snapshot                 )                        {
-  const byIdentity = new Map                  (); // identity -> stageId[]
+  const byIdentity = new Map                  ();
   for (const [id, stage] of snapshot.stages) {
     const key = stageIdentity(stage, snapshot);
     const ids = byIdentity.get(key);
@@ -134,24 +119,35 @@ export function matchStages(baseSnap                 , candSnap
   const pairs                                                              = [];
   const matchedIdentities = new Set        ();
   const collisionIdentities = new Set        ();
-  // A collision is a single-run property: any identity that more than one stage
-  // in the SAME run normalizes to. Record them per-run, independent of whether
-  // the other run shares that identity (a colliding stage with no counterpart
-  // still makes the run ambiguous).
+  // A collision is a single-run property: more than one stage in the SAME run
+  // shares an identity. Recorded per-run regardless of whether the other run
+  // shares it too. Some collisions get resolved into `pairs` below and some
+  // don't, so this set means "was ambiguous", not "stayed unpaired".
   for (const idx of [baseIdx, candIdx])
     for (const [identity, ids] of idx) if (ids.length > 1) collisionIdentities.add(identity);
-  // Pair only identities that map to exactly one stage on BOTH sides.
-  // Deterministic order: sort identities lexically.
+  // An identity colliding equally on both sides has no genuine ambiguity about
+  // *count*, so pair its stages off positionally by sorted id rather than
+  // dropping them. This is exact when the identity actually distinguishes
+  // stages (e.g. self-comparing a run: every stage matches itself). It's a
+  // best-effort guess when the identity is coarse (no SQL/attribution) and
+  // the two sides are genuinely different runs -- two unrelated same-named
+  // stages could get cross-paired. Accepted tradeoff: dropping them instead
+  // would also sacrifice the exact-self-comparison case, which matters more.
   for (const identity of [...baseIdx.keys()].sort()) {
-    if (!candIdx.has(identity)) continue;
+    const candIds = candIdx.get(identity);
+    if (!candIds) continue;
     const baseIds = baseIdx.get(identity) ;
-    const candIds = candIdx.get(identity) ;
-    if (baseIds.length > 1 || candIds.length > 1) continue; // collision already recorded above
-    pairs.push({ identity, baseId: baseIds[0], candId: candIds[0] });
+    if (baseIds.length !== candIds.length) continue;
+    const sortedBaseIds = [...baseIds].sort((a, b) => a - b);
+    const sortedCandIds = [...candIds].sort((a, b) => a - b);
+    for (let i = 0; i < sortedBaseIds.length; i++) {
+      pairs.push({ identity, baseId: sortedBaseIds[i], candId: sortedCandIds[i] });
+    }
     matchedIdentities.add(identity);
   }
   const total = baseSnap.stages.size + candSnap.stages.size;
-  const coverage = total === 0 ? 0 : (2 * pairs.length) / total;
+  // Both runs stage-less: nothing to compare, not "nothing matched".
+  const coverage = total === 0 ? 1 : (2 * pairs.length) / total;
   return { pairs, matchedIdentities, collisionIdentities, coverage };
 }
 
@@ -329,18 +325,19 @@ function namesConflict(a                     , b                     )
   return a?.name != null && b?.name != null && a.name !== b.name;
 }
 
-// A pair's skew ratio is null when the stage's duration wasn't measurable
-// (see `ratio` below), matching CompareRunsResult['stageSkew']'s nullable
-// baseline/candidate/delta fields.
+// A pair's skew ratio is null when the stage's duration wasn't measurable (see
+// `stageSkewRatio`). `baseId`/`candId` are carried through so a consumer can
+// key rows uniquely: two pairs can share one `identity` (a same-run collision
+// resolved positionally in matchStages), but never the same baseId+candId.
 function stageSkewDeltas(
   baseSnap                 ,
   candSnap                 ,
   match                                                                        ,
-)                                                                                                       {
+)                                                                                                                                       {
   return match.pairs.map((p) => {
     const b = stageSkewRatio(baseSnap.stages.get(p.baseId) );
     const c = stageSkewRatio(candSnap.stages.get(p.candId) );
-    return { identity: p.identity, baseline: b, candidate: c, delta: b != null && c != null ? c - b : null };
+    return { identity: p.identity, baseId: p.baseId, candId: p.candId, baseline: b, candidate: c, delta: b != null && c != null ? c - b : null };
   }).sort((x, y) => x.identity < y.identity ? -1 : x.identity > y.identity ? 1 : 0);
 }
 
@@ -379,15 +376,13 @@ function stageList(snap                 )
   return [...snap.stages].map(([id, s]) => ({ id, name: s.name ?? `Stage ${id}`, metrics: perStageMetrics(s) }));
 }
 
-// Shared by every buildComparison caller below: captureSnapshot only ever
-// copies this into a fresh Map internally, never mutates the caller's
-// reference, so one shared empty Map is safe in place of a fresh allocation
-// per call.
+// captureSnapshot copies this into a fresh Map internally and never mutates the
+// caller's reference, so one shared empty Map is safe here.
 const EMPTY_TASK_DATA = new Map                 ();
 
 // Wraps the analyze()-to-compareRuns() snapshot-building sequence shared by
 // the CLI's --baseline path and mcp-tools.ts's compareRuns tool: both need
-// captureSnapshot (with an empty taskDataCache — the interactive drill-down
+// captureSnapshot (with an empty taskDataCache, the interactive drill-down
 // cache, never read by compareRuns/matchStages/metricDeltas/findingsDelta)
 // for each side before diffing them.
 export function buildComparison(
@@ -400,6 +395,14 @@ export function buildComparison(
   );
 }
 
+// matchStages' coverage is a Dice coefficient: (2 * pairs.length) / (baseCount
+// + candCount). Below 0.5, more than half of each run's stages went unpaired,
+// so the stage-level rows (stageSkew, baseStages/candStages) mostly show
+// unrelated work side by side rather than the same stage before/after -- the
+// comparison is dominated by guesswork, not genuine pairing. 0.5 is thus the
+// natural midpoint for "more matched than not," not an arbitrary tuning knob.
+const LOW_COVERAGE_THRESHOLD = 0.5;
+
 export function compareRuns(
   baseline                                              ,
   candidate                                              ,
@@ -410,10 +413,21 @@ export function compareRuns(
   // is the normal way to label an A/B experiment, so a mismatch must not block
   // the (matching-free, name-independent) deltas. Surface it as `low` instead.
   const namesDiffer = namesConflict(baseSnap.app, candSnap.app);
+  // Coverage is the other half of the signal: identical names on two runs that
+  // barely share any stages are just as misleading as differing names on two
+  // runs that match well, so either condition alone drops confidence to `low`.
+  const lowCoverage = match.coverage < LOW_COVERAGE_THRESHOLD;
+  const reason = namesDiffer && lowCoverage
+    ? `Run names differ and only ${(match.coverage * 100).toFixed(0)}% of stages matched, so deltas may compare different work.`
+    : namesDiffer
+    ? 'Run names differ, so deltas may compare different work.'
+    : lowCoverage
+    ? `Only ${(match.coverage * 100).toFixed(0)}% of stages matched between runs, so per-stage rows mostly compare unrelated work.`
+    : null;
   return {
     baselineLabel: baseline.label, candidateLabel: candidate.label,
-    confidence: namesDiffer ? 'low' : 'ok',
-    reason: namesDiffer ? 'Run names differ, so deltas may compare different work.' : null,
+    confidence: namesDiffer || lowCoverage ? 'low' : 'ok',
+    reason,
     matchedCoverage: match.coverage,
     metrics: metricDeltas(baseSnap, candSnap),
     findings: findingsDelta(baseSnap, candSnap),
@@ -440,7 +454,7 @@ function renderFindingsSection(title        , findings                    )
 export function renderComparisonMarkdown(comparison                   )         {
   const lines = ['', '## Comparison to baseline', ''];
   if (comparison.confidence === 'low') {
-    lines.push(`- confidence: low — ${comparison.reason}`);
+    lines.push(`- confidence: low, ${comparison.reason}`);
     lines.push('');
   }
   lines.push(`- matched stage coverage: ${(comparison.matchedCoverage * 100).toFixed(1)}%`);

package/vendor-core/scaling-sim.js

@@ -1,4 +1,4 @@
-// §4 What-if executor-scaling simulator (SparkLens ExecutorWallclockAnalyzer).
+// §4 What-if executor-scaling simulator.
 // DESIGN SPIKE: the makespan model is a first-cut approximation and is NOT
 // validated against ground truth (a single event log only ever observes one
 // scale). Every prediction ships with a Model Error confidence indicator.
@@ -42,13 +42,11 @@ export function simulateScaling({ app, stages, runAggregates, executorsAdded }
                                
   {
   const perStage = runAggregates?.perStage ?? {};
-  // `app ?? {}`: computeTotalCores just falls back to the executor-derived core sum when
-  // `resources` is absent, and every caller here already tolerates a null `app` (malformed
-  // logs with no ApplicationStart); `app!` would crash on `app.resources` in that case.
+  // `app ?? {}`: computeTotalCores falls back to the executor-derived core sum
+  // when `resources` is absent, and a null `app` (malformed log, no
+  // ApplicationStart) is real here; `app!` would crash on `app.resources`.
   // `executorsAdded` cast: computeTotalCores only reads `totalCores`, present on
-  // ExecutorAddedEvent (the only kind real callers pass here) but not ExecutorRemovedEvent,
-  // so the union as a whole is a structural mismatch against computeTotalCores's
-  // `{totalCores?}` shape.
+  // ExecutorAddedEvent (the only kind passed here) but not on the union type.
   const baselineCores = computeTotalCores(app ?? {}, executorsAdded                                  );
   const wc = computeWallClock(app, stages                                                               );
   const observedActiveMs = wc.stagesActive;

package/vendor-core/session-snapshot.js

@@ -67,7 +67,7 @@ export function applySnapshot(
   appModel.jobs = new Map(snapshot.jobs);
   appModel.runAggregates = snapshot.runAggregates ?? null;
   // Fail-closed on unknown schema versions: a future incompatible ledger is
-  // never trusted as V1 data (fixes: schemaVersion must be validated at read).
+  // never trusted as V1 data.
   appModel.evidenceAvailability = isSupportedEvidenceAvailability(snapshot.evidenceAvailability)
     ? snapshot.evidenceAvailability
     : null;

package/vendor-core/shs-fetch.js

@@ -31,10 +31,9 @@ export function sniffCodec(bytes            )
 // it (fzstd's `decompress()`, fflate's `gunzipSync()`) allocates that entire
 // size as a single ArrayBuffer up front, which can exceed what the browser
 // will allocate for a multi-GB event log ("Array buffer allocation failed").
-// fflate's Gunzip and fzstd's Decompress are untyped vendor JS (plain
-// prototype classes, not `class` declarations), so TS can't infer a
-// construct signature for them; this local shape is just enough to type
-// the two call sites below without touching the vendored files.
+// fflate's Gunzip and fzstd's Decompress are untyped vendor JS, so TS can't
+// infer a construct signature for them; this local shape types the call sites
+// without touching the vendored files.
                                                                            
                                                                                            
 
@@ -68,8 +67,7 @@ export async function runParseFromUrl(
   state             ,
   { fetchImpl = fetch, emit = (msg         ) => self.postMessage(msg) }                                              = {}
 )                {
-  // buildProxyRequestUrl is still-untyped JS (./shs-request.js, a plain
-  // untyped .js sibling); `request`'s real shape isn't pinned down here.
+  // buildProxyRequestUrl is untyped JS; `request`'s real shape isn't pinned here.
   const url = buildProxyRequestUrl(request                                              );
 
   let res;

package/vendor-core/shs-load.js

@@ -61,12 +61,10 @@ async function readArchiveBytes(upstream          , maxBytes        , idleTimeou
   const reader = upstream.body .getReader();
   // Grown geometrically instead of preallocated at `maxBytes`: a typical
   // archive is nowhere near the cap, so starting small (or at the declared
-  // content-length, when trustworthy) and doubling on demand keeps peak
-  // memory close to the actual download size. Chunks are copied straight into
-  // this buffer as they arrive rather than buffered in an array and copied
-  // once at the end, so the previous ~2x-of-total peak (chunk array + a
-  // freshly allocated same-size output buffer, both live during the final
-  // copy) no longer happens.
+  // content-length, when trustworthy) and doubling on demand keeps peak memory
+  // close to the actual download size. Chunks are copied straight into this
+  // buffer as they arrive rather than buffered in an array and copied once at
+  // the end, which would keep ~2x-of-total live during the final copy.
   let buf = new Uint8Array(Number.isFinite(declared) && declared > 0 ? Math.min(declared, maxBytes) : 65536);
   let total = 0;
   for (;;) {
@@ -96,11 +94,9 @@ async function readArchiveBytes(upstream          , maxBytes        , idleTimeou
   return buf.subarray(0, total);
 }
 
-// ./proxy.js is a plain, untyped .js file whose exported fetchShsEventLog
-// TS can only infer a loose shape for. Its real (verified by reading
-// proxy.js) contract is this
-// discriminated union (either branch, never a mix), so it's asserted here
-// once at the boundary rather than widening every downstream read.
+// fetchShsEventLog is untyped JS; its real contract is this discriminated union
+// (either branch, never a mix), asserted here once at the boundary rather than
+// widening every downstream read.
                                                                                       
 
 export async function resolveFromShs(
@@ -108,7 +104,7 @@ export async function resolveFromShs(
   appId        ,
   attemptId         ,
   opts                                                                                 = {},
-)                    {
+)                                                        {
   const { fetchImpl = fetch, maxArchiveBytes = DEFAULT_MAX_ARCHIVE_BYTES, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS } = opts;
   const validated = validateShsRequest({ baseUrl: shsBaseUrl, appId, attemptId: attemptId ?? '' });
   if (!validated.request) throw mcpError('access-or-upstream-failure', 'Invalid SHS request parameters.');
@@ -117,5 +113,5 @@ export async function resolveFromShs(
   const zipBytes = await readArchiveBytes(fetched.upstream, maxArchiveBytes, idleTimeoutMs);
   const { appModel, skippedLines } = await collectShsAppModel(zipBytes);
   appModel.evidenceAvailability = deriveEvidenceAvailability(appModel, { skippedLines });
-  return appModel;
+  return { appModel, skippedLines };
 }

package/vendor-core/shs-request.js

@@ -19,7 +19,7 @@ function trimString(value) {
   return typeof value === 'string' ? value.trim() : '';
 }
 
-function normalizeBaseUrl(value) {
+export function normalizeBaseUrl(value) {
   const baseUrl = trimString(value);
   if (!baseUrl || baseUrl.includes('?') || baseUrl.includes('#')) return null;
 

package/vendor-core/stage-quantiles.js

@@ -1,3 +1,5 @@
+                                                            
+
 // Single source of truth for the packed per-task numeric array: FIELDS (offset
 // constants), TASK_FIELD_NAMES (display labels), and the finalizeStage hot-loop
 // push order are all derived from this list. `prop` is the task record's real
@@ -22,6 +24,10 @@ const TASK_FIELD_DESCRIPTORS                        = [
 
 const STRIDE = TASK_FIELD_DESCRIPTORS.length;
 
+// Mirrors event-handlers.ts's MAX_TASK_SAMPLES (kept as a separate constant to avoid a circular
+// value import between the two modules: see that file's comment).
+const MAX_TASK_SAMPLES = 20;
+
 export const FIELDS = Object.freeze({
   ...Object.fromEntries(TASK_FIELD_DESCRIPTORS.map((d, i) => [d.key, i])),
   STRIDE,
@@ -67,6 +73,7 @@ export function finalizeStage(
   const hostStats = new Map();
   const executorStats = new Map();
   const failureReasons = new Map();
+  const failedTaskSamples                     = [];
   const localityStats = new Map();
   let peakExecutionMemoryMax = 0;
 
@@ -75,6 +82,12 @@ export function finalizeStage(
     if (t.failed) {
       failedTasks++;
       if (t.reason) failureReasons.set(t.reason, (failureReasons.get(t.reason) ?? 0) + 1);
+      if (failedTaskSamples.length < MAX_TASK_SAMPLES) {
+        failedTaskSamples.push({
+          taskId: t.taskId, attemptNumber: t.attemptNumber, host: t.host, executorId: t.executorId,
+          reason: t.reason, peakExecMem: t.peakExecMem, memSpilled: t.memSpilled, shuffleWrite: t.shuffleWrite,
+        }                               );
+      }
     }
     if (t.speculative) speculativeTasks++;
     if (t.host) {
@@ -148,6 +161,7 @@ export function finalizeStage(
   const data                          = {
     ...stage,
     hostStats: hostStatsArr, executorStats: executorStatsArr, failureReasons: failureReasonsArr, localityStats: localityStatsArr, stragglerCount,
+    failedTaskSamples,
     peakExecutionMemoryMax,
     taskDurationP50: p50,
     taskDurationP95: p95,