sparkforensics-cli 0.1.0

package/vendor-core/recommendation-rollup.js

@@ -0,0 +1,197 @@
+// Explicit .ts extensions (unlike a bundler, plain Node's ESM resolver -
+// the runtime bin/sparkforensics-analyze.mjs and src/mcp-tools.ts both run
+// under, now that src/evidence-report.ts imports this module too - requires
+// the exact specifier, not an extensionless one a bundler would resolve for
+// you): this file's prior only consumer, src/view/widgets/FixTheseFirst.tsx,
+// went through Vite, which tolerates the extensionless form, so this never
+// surfaced until the CLI/MCP path started importing it.
+import { mergeIntervals } from './wall-clock.js';
+import { worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
+                                                                    
+
+                         
+                       
+                       
+ 
+
+/** The cross-finding grouping layer the spec calls for: given every stage
+ * touched by a group of findings, collect their intervals and merge once
+ * across the combined set, instead of summing each finding's own
+ * already-clipped wallClock figure (which double-counts concurrent stages
+ * across different findings of the same detector type). Reuses the generic
+ * mergeIntervals primitive (src/wall-clock.ts) rather than widening the
+ * single-finding-scoped estimateMultiStage/unionMs in src/occupancy.ts. */
+export function computeStageUnionMs(
+  stageIds          ,
+  stages                            ,
+)         {
+  const intervals                     = [];
+  for (const id of stageIds) {
+    const stage = stages.get(id);
+    if (!stage) continue;
+    // Skip half-open stages rather than defaulting a missing bound to 0: an
+    // incomplete/truncated log (the case `incompleteRun` flags) leaves
+    // `completedAt` unset, and coercing it to 0 would contribute a hugely
+    // negative interval and a nonsense negative recoverable-time figure.
+    // Same filter `computeWallClock` (src/wall-clock.ts) already applies.
+    if (stage.submittedAt == null || stage.completedAt == null) continue;
+    intervals.push([stage.submittedAt, stage.completedAt]);
+  }
+  return mergeIntervals(intervals).reduce((sum, [a, b]) => sum + (b - a), 0);
+}
+
+                         
+                                                                                                                            
+                                                                                                                    
+                                                                                                                                  
+
+function groupByType(findings           )                         {
+  const groups = new Map                   ();
+  for (const finding of findings) {
+    const group = groups.get(finding.type) ?? [];
+    group.push(finding);
+    groups.set(finding.type, group);
+  }
+  return groups;
+}
+
+function stageIdsOf(finding         )           {
+  if (finding.stageIds) return finding.stageIds;
+  if (finding.stageId != null) return [finding.stageId];
+  return [];
+}
+
+function buildTimeGroup(
+  type        ,
+  findings           ,
+  stages                                                             ,
+)              {
+  const stageIds = [...new Set(findings.flatMap(stageIdsOf))];
+  const naiveSum = findings.reduce((sum, f) => sum + (f.impactEstimate?.wallClock?.high ?? 0), 0);
+  // Only cap to stage union if there are stages to compare; stageless findings use naive sum.
+  const recoverableMsHigh = stageIds.length > 0
+    ? Math.min(naiveSum, computeStageUnionMs(stageIds, stages))
+    : naiveSum;
+  return { kind: 'time', type, findingCount: findings.length, stageCount: stageIds.length, recoverableMsHigh, findings };
+}
+
+function buildResourceGroup(type        , findings           )              {
+  const unit = findings[0].impactEstimate .rawWaste .unit;
+  const total = findings.reduce((sum, f) => sum + (f.impactEstimate?.rawWaste?.value ?? 0), 0);
+  return { kind: 'resource', type, findingCount: findings.length, unit, total, findings };
+}
+
+function buildCountGroup(type        , findings           )              {
+  const byImpactBand                                      = {};
+  for (const finding of findings) {
+    byImpactBand[finding.impactBand] = (byImpactBand[finding.impactBand] ?? 0) + 1;
+  }
+  return { kind: 'count', type, findingCount: findings.length, byImpactBand, findings };
+}
+
+const KIND_ORDER                                      = { time: 0, resource: 1, count: 2 };
+
+export function buildRecommendationRollup(
+  findings           ,
+  stages                                                             ,
+)                {
+  const groups                = [];
+  for (const [type, typeFindings] of groupByType(findings)) {
+    const timeFindings = typeFindings.filter((f) => f.impactEstimate?.wallClock != null);
+    const resourceFindings = typeFindings.filter(
+      (f) => f.impactEstimate?.wallClock == null && f.impactEstimate?.rawWaste != null,
+    );
+    const countFindings = typeFindings.filter(
+      (f) => f.impactEstimate?.wallClock == null && f.impactEstimate?.rawWaste == null,
+    );
+    if (timeFindings.length > 0) groups.push(buildTimeGroup(type, timeFindings, stages));
+    // Group resource findings by (type, unit) to prevent mixing incompatible units.
+    if (resourceFindings.length > 0) {
+      const byUnit = new Map                         ();
+      for (const finding of resourceFindings) {
+        const unit = finding.impactEstimate .rawWaste .unit;
+        const group = byUnit.get(unit) ?? [];
+        group.push(finding);
+        byUnit.set(unit, group);
+      }
+      for (const resourceGroup of byUnit.values()) {
+        groups.push(buildResourceGroup(type, resourceGroup));
+      }
+    }
+    if (countFindings.length > 0) groups.push(buildCountGroup(type, countFindings));
+  }
+  return groups.sort((a, b) => {
+    const kindDelta = KIND_ORDER[a.kind] - KIND_ORDER[b.kind];
+    if (kindDelta !== 0) return kindDelta;
+    if (a.kind === 'time' && b.kind === 'time') return b.recoverableMsHigh - a.recoverableMsHigh;
+    // resource/count groups carry no shared magnitude to rank by (spec section
+    // 5 forbids comparing incompatible units); break ties by worst impact band
+    // instead, the same tie-break FixTheseFirst used pre-consolidation.
+    const aImpactBand = IMPACT_BAND_ORDER[worstImpactBand(a.findings) ?? 'info'] ?? 9;
+    const bImpactBand = IMPACT_BAND_ORDER[worstImpactBand(b.findings) ?? 'info'] ?? 9;
+    return aImpactBand - bImpactBand;
+  });
+}
+
+/** True when a finding is a candidate for the "fix these first" ranking at
+ * all. Shared by `src/evidence-report.ts` (CLI/MCP path) and
+ * `src/view/widgets/FixTheseFirst.tsx` (the dashboard widget) so these two
+ * hardcoded exclusions can't drift between the two surfaces (they used to be
+ * duplicated, near-verbatim, in both files). Doesn't check
+ * `REGISTRY[finding.type] != null` (FixTheseFirst.tsx's own extra check):
+ * `REGISTRY` lives in a `.tsx` file, not importable from this core module;
+ * FixTheseFirst.tsx layers that extra check on top of this one instead. */
+export function isEligible(finding         )          {
+  // incompleteRun is a pipeline-completeness signal, not an addressable fix:
+  // hardcoded here by type, not read from Detector.fixEffort. That field
+  // exists only to back its own contract test today (tests/fix-effort.test.js);
+  // nothing in ranking/exclusion logic reads it.
+  if (finding.type === 'incompleteRun') return false;
+  // memoryUtilization's memoryBand/dataUnavailable variant reports a missing
+  // -evidence caveat (spark.eventLog.logStageExecutorMetrics=true not on for
+  // this run), not an optimization: the exact same fact already lives in the
+  // Evidence availability ledger's own `executorMetrics` entry
+  // (src/evidence-availability.ts), so it belongs there, not here.
+  if (finding.type === 'memoryUtilization' && finding.variant === 'memoryBand' && finding.dataUnavailable) return false;
+  return true;
+}
+
+/** Ranking tier for a single finding: time-based findings (a real `wallClock`
+ * claim) are the only ones whose magnitudes share a unit, so they are the
+ * only ones ranked numerically. `resourceOnly` findings carry `rawWaste` in
+ * whatever unit their detector emitted, which the spec (section 5) forbids
+ * comparing against a time figure or against another resourceOnly finding's
+ * own raw magnitude; they rank by impact band instead, below every time-based
+ * finding. Informational findings (no magnitude at all) rank last, also by
+ * impact band. */
+const TIER_TIME = 0;
+const TIER_RESOURCE = 1;
+const TIER_INFORMATIONAL = 2;
+
+function tierOf(finding         )         {
+  const estimate = finding.impactEstimate;
+  if (estimate?.wallClock) return TIER_TIME;
+  if (estimate?.rawWaste) return TIER_RESOURCE;
+  return TIER_INFORMATIONAL;
+}
+
+/** Ranks a group's members so the "representative" finding (the one whose
+ * action label/tag donate to the group's row) is always the highest-impact
+ * one, not just the first one `buildRecommendationRollup` happened to
+ * collect. Used to pick each type-group's own highest-impact member and to
+ * order that group's expanded list; cross-group ordering is
+ * `buildRecommendationRollup`'s own job, not this function's. */
+export function rankFindings(findings           )            {
+  return [...findings].sort((a, b) => {
+    const tierDelta = tierOf(a) - tierOf(b);
+    if (tierDelta !== 0) return tierDelta;
+    if (tierOf(a) === TIER_TIME) {
+      // Same unit (ms) on both sides: the one comparison that is meaningful.
+      // .high, matching deriveImpactBand (src/impact-band.ts) and
+      // triage-target.ts, so this ordering, the badge color, and the "click
+      // to investigate" target all agree on the same figure.
+      return b.impactEstimate .wallClock .high - a.impactEstimate .wallClock .high;
+    }
+    return (IMPACT_BAND_ORDER[a.impactBand] ?? 9) - (IMPACT_BAND_ORDER[b.impactBand] ?? 9);
+  });
+}

package/vendor-core/redact.js

@@ -0,0 +1,175 @@
+// Identifier redaction for the portable evidence report. Replaces the
+// application id and every host name with stable pseudonyms (`app-1`,
+// `host-1`, ...) so a report can be shared outside the environment that
+// produced it without leaking infrastructure identity. Opt-in (default OFF)
+// at the call site; raw task records are excluded from the report regardless.
+//
+// Deterministic (sorted assignment), idempotent (pseudonyms map to themselves),
+// and non-mutating (returns a fresh, deep-copied tree).
+
+// Host / IP identifier patterns. Used to enumerate host names that surface only
+// inside free text: recommendation strings, `stageFailed`'s failure-reason
+// value, SQL relation/node names, never as a structured `host` field, so
+// redaction reaches those residuals too. Pseudonyms (`host-1`) match neither
+// pattern, keeping the scan idempotent.
+const HOST_PATTERNS = [
+  // EC2-style ip-10-1-2-3 with an optional dotted domain (ip-10-1-2-3.ec2.internal).
+  // Each domain label must start with an alphanumeric, so a trailing sentence
+  // period ("… bad node ip-10-1-2-3.") is left out of the match.
+  /\bip(?:-\d{1,3}){4}(?:\.[a-z0-9][a-z0-9-]*)*/gi,
+  /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, // bare IPv4: 10.1.2.3
+];
+
+// Spark application id (e.g. application_1690000000000_0001). Besides
+// redactReport's structured summary.app.id field, an app id can also surface
+// as a residual free-text token (e.g. embedded in a finding's evidence or
+// recommendation string) — same as a host/IP token embedded in a stage name
+// or plan detail. redactComparison has no structured app-id field at all
+// (baselineLabel/candidateLabel are caller-supplied labels, not Spark app
+// ids), so free text is its *only* source of app ids.
+const APP_ID_PATTERNS = [/\bapplication_\d{10,}_\d+\b/g];
+
+// Walk every string in the tree once, collecting matches for each `{ patterns,
+// out }` sink. One shared traversal for every token kind (instead of one
+// traversal per kind) keeps redactComparison's dual host+app-id scan the same
+// cost as the single-kind scan redactReport/redactAppIdentity already do.
+function scanTokens(node         , sinks                                                 )       {
+  if (typeof node === 'string') {
+    for (const { patterns, out } of sinks) {
+      for (const re of patterns) {
+        const found = node.match(re);
+        if (found) for (const m of found) out.add(m);
+      }
+    }
+    return;
+  }
+  if (Array.isArray(node)) {
+    for (const n of node) scanTokens(n, sinks);
+    return;
+  }
+  if (node && typeof node === 'object') {
+    for (const v of Object.values(node)) scanTokens(v, sinks);
+  }
+}
+
+// Walk every string in the tree, collecting host/IP tokens into `hosts`.
+function scanHostTokens(node         , hosts             )       {
+  scanTokens(node, [{ patterns: HOST_PATTERNS, out: hosts }]);
+}
+
+// Known locations of the identifiers, so we don't have to guess which strings
+// are sensitive: the app id lives at summary.app.id; host names live on each
+// finding's `host` field. The real report nests that host under `evidence.host`
+// (host is not a first-class report column), so read both shapes, plus a
+// free-text scan for hosts/IPs and app ids that appear only inside string
+// values (e.g. a finding's evidence/recommendation text).
+                            
+                                                                     
+                                                                        
+                                                                      
+                                                                           
+                                             
+                                                                    
+ 
+
+function collectIds(report                  )                                              {
+  const appIds = new Set        ();
+  const hosts = new Set        ();
+  const appId = report?.summary?.app?.id;
+  if (typeof appId === 'string' && appId.length > 0) appIds.add(appId);
+  for (const f of report?.findings ?? []) {
+    for (const h of [f?.host, f?.evidence?.host]) {
+      if (typeof h === 'string' && h.length > 0) hosts.add(h);
+    }
+  }
+  // Free-text scan for both host/IP tokens and app-id tokens: an app id can
+  // surface in a finding's evidence/recommendation text (e.g. "retry app
+  // application_1690000000000_0001 failed") same as redactComparison's scan.
+  scanTokens(report, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
+  return { appIds, hosts };
+}
+
+// Numeric-aware sorted assignment => deterministic numbering that is stable
+// across passes. A lexicographic sort orders `host-1, host-10, host-11, host-2`
+// so a second pass over already-pseudonymized ids would re-slot `host-10`→`2`
+// at >=10 items and break idempotency; a numeric-aware sort keeps `host-2`
+// before `host-10`, so each pseudonym maps back to itself.
+function buildMap(ids             , prefix        )                      {
+  const map = new Map                ();
+  [...ids]
+    .sort((a, b) => a.localeCompare(b, 'en', { numeric: true }))
+    .forEach((id, i) => map.set(id, `${prefix}-${i + 1}`));
+  return map;
+}
+
+// Deep-replace every string occurrence of each identifier across the whole
+// tree (covers ids embedded in free-text recommendations, not just the
+// canonical fields). Longest-first so no identifier is a prefix-shadow of
+// another. Returns a fresh tree, never mutates the input.
+function deepReplace(node         , replacements                         )          {
+  if (typeof node === 'string') {
+    let s = node;
+    for (const [from, to] of replacements) s = s.split(from).join(to);
+    return s;
+  }
+  if (Array.isArray(node)) return node.map((n) => deepReplace(n, replacements));
+  if (node && typeof node === 'object') {
+    const out                          = {};
+    for (const [k, v] of Object.entries(node)) out[k] = deepReplace(v, replacements);
+    return out;
+  }
+  return node;
+}
+
+// Shared by every redact* export below: turns collected app-id/host sets into
+// the longest-first replacement list and applies it. Centralizes the
+// buildMap+sort+deepReplace sequence so each narrower redact* function only
+// has to say what it collects, not how replacement is carried out.
+function applyReplacements   (node   , ids                                              )    {
+  const merged                          = [
+    ...(ids.appIds ? buildMap(ids.appIds, 'app') : []),
+    ...buildMap(ids.hosts, 'host'),
+  ];
+  // Replace longer identifiers first to avoid partial-substring clobbering.
+  merged.sort((a, b) => b[0].length - a[0].length);
+  return deepReplace(node, merged)     ;
+}
+
+export function redactReport                            (report   )    {
+  const { appIds, hosts } = collectIds(report);
+  return applyReplacements(report, { appIds, hosts });
+}
+
+// Narrow counterpart to redactReport(), for getRunSummary()'s standalone app
+// object (no findings tree to walk). There's exactly one app id here, so no
+// Set/Map/sort is needed for it; name/sparkVersion still go through the
+// shared host/IP scan-and-replace since either can carry a host token as
+// free text.
+export function redactAppIdentity(
+  app                                                                         ,
+)                                                                          {
+  const hosts = new Set        ();
+  scanHostTokens(app.name, hosts);
+  scanHostTokens(app.sparkVersion, hosts);
+  return {
+    id: typeof app.id === 'string' && app.id.length > 0 ? 'app-1' : app.id,
+    name: applyReplacements(app.name, { hosts }),
+    sparkVersion: applyReplacements(app.sparkVersion, { hosts }),
+  };
+}
+
+// Run-comparison counterpart: no single app-id *field* to pseudonymize
+// (baselineLabel/candidateLabel are caller-supplied labels, not Spark app
+// ids), but stage names surface throughout the tree (FindingsDeltaRow.stages,
+// baseStages/candStages[].name) and, like the evidence report's findings, can
+// carry a host/IP token or an app id as free text (e.g. `collect at
+// application_1690000000000_0001 worker-10-1-2-3.scala:45`). Scans the whole
+// comparison tree rather than enumerating those fields individually, so a
+// future CompareRunsResult field carrying free text is covered without this
+// function needing to change.
+export function redactComparison   (comparison   )    {
+  const hosts = new Set        ();
+  const appIds = new Set        ();
+  scanTokens(comparison, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
+  return applyReplacements(comparison, { appIds, hosts });
+}

package/vendor-core/rolling-log-reassembly.js

@@ -0,0 +1,52 @@
+// Rolling event-log directory/zip member-name reassembly. Pulled out of
+// shs-fetch.ts (Task: bundle-size fix) so DropZone.tsx's drag-drop path can
+// import just this reassembly logic without pulling in the Zod schemas and
+// vendored zstd/gzip decompressors that the rest of shs-fetch.ts (and the
+// wider parser-worker.ts module graph) depend on. Keep this file free of any
+// import beyond plain JS/TS built-ins.
+
+export function naturalCompare(a        , b        )         {
+  const tokenize = (s        ) => s.match(/(\d+)|(\D+)/g) ?? [s];
+  const ax = tokenize(a), bx = tokenize(b);
+  const len = Math.max(ax.length, bx.length);
+  for (let i = 0; i < len; i++) {
+    const av = ax[i] ?? '', bv = bx[i] ?? '';
+    if (av === bv) continue;
+    const an = Number(av), bn = Number(bv);
+    if (!Number.isNaN(an) && !Number.isNaN(bn)) return an - bn;
+    return av < bv ? -1 : 1;
+  }
+  return 0;
+}
+
+// Reassemble a rolling `eventlog_v2_*` directory's (or SHS zip's) member
+// names into the ordered list that should actually be parsed: drop the
+// zero-byte `appstatus_*` completion marker, drop every non-compact
+// `events_*` file at or below the most recent `*.compact` file's index
+// (already merged into it, so reading them again double-counts events), and
+// sort the remainder in ascending numeric-index order. Throws if the
+// resulting index sequence has a gap (a missing roll file).
+export function reassembleRollingEntries(names          )           {
+  const withoutMarker = names.filter(n => !n.toLowerCase().startsWith('appstatus'));
+  const eventNames = withoutMarker.filter(n => /^events_\d+_/.test(n));
+
+  const indexOf = (n        ) => parseInt(n.match(/^events_(\d+)_/) [1], 10);
+
+  const compactIndices = eventNames.filter(n => n.endsWith('.compact')).map(indexOf);
+  const highestCompactIndex = compactIndices.length > 0 ? Math.max(...compactIndices) : -1;
+
+  const kept = eventNames.filter(n => {
+    const index = indexOf(n);
+    return n.endsWith('.compact') ? index === highestCompactIndex : index > highestCompactIndex;
+  });
+
+  const sorted = kept.sort(naturalCompare);
+  const indices = sorted.map(indexOf);
+  for (let i = 1; i < indices.length; i++) {
+    if (indices[i] !== indices[i - 1] + 1) {
+      throw new Error(`Rolling event-log directory is missing file(s) between index ${indices[i - 1]} and ${indices[i]}.`);
+    }
+  }
+
+  return sorted;
+}

package/vendor-core/run-aggregates.js

@@ -0,0 +1,44 @@
+// Whole-run aggregation pass. Runs INSIDE parser-worker.js over the retained
+// per-task launch/finish timestamps in taskStore, so raw task data never
+// reaches the main thread: only the compact summary below is posted. Pure and
+// worker-agnostic so it is unit-testable without a Worker.
+//
+// Uses computeCoreTimeSeries in coreCount mode (O(n log n)) to derive the
+// busy-core histogram; the time-bucket mode is intentionally NOT used here
+// because it is O(buckets × segments) and a whole run can hold millions of
+// tasks (see core-time-series.js PERF note).
+
+import { FIELDS } from './stage-quantiles.js';
+import { computeCoreTimeSeries } from './core-time-series.js';
+
+export function computeRunAggregates(taskStore                           )   
+                          
+                     
+                              
+                                                                                
+  {
+  const intervals                                            = [];
+  const perStage                                                                      = {};
+  for (const [stageId, arr] of taskStore) {
+    const taskCount = arr.length / FIELDS.STRIDE;
+    let totalTaskDurationSum = 0;
+    for (let i = 0; i < taskCount; i++) {
+      const base = i * FIELDS.STRIDE;
+      const launch = arr[base + FIELDS.LAUNCH_TIME];
+      const finish = arr[base + FIELDS.FINISH_TIME];
+      totalTaskDurationSum += arr[base + FIELDS.DURATION];
+      if (finish > launch) intervals.push({ launch, finish });
+    }
+    perStage[stageId] = { totalTaskDurationSum, taskCount };
+  }
+
+  // bucketBy: 'coreCount' always yields the coreCount branch of the union;
+  // narrow explicitly since the callee's return type doesn't correlate to
+  // the literal `bucketBy` argument.
+  const { histogram } = computeCoreTimeSeries(intervals, { bucketBy: 'coreCount' })                                              ;
+  let busyCoreMs = 0;
+  for (let k = 0; k < histogram.length; k++) busyCoreMs += k * histogram[k];
+  const peakConcurrentCores = histogram.length > 0 ? histogram.length - 1 : 0;
+
+  return { coreHistogram: histogram, busyCoreMs, peakConcurrentCores, perStage };
+}