sparkforensics-cli 0.1.0 → 0.2.0

package/vendor-core/format-utils.js

@@ -11,15 +11,11 @@ const BOTTLENECK_WIDGET                         = {
   coldStart: 'executor-timeline', utilization: 'executor-timeline', speculationWaste: 'executor-timeline',
 };
 
-// Cross-widget stage recurrence: how many *distinct board widgets* (not raw
-// catalog entries: skew+straggler on one stage still count as one widget,
-// task-skew) flag a given stage. Computed once from the full catalog each
-// widget already receives, so no new parameter threads through the
-// dashboard-renderer render loop.
-// `stageId` widened to also accept `undefined` (Task 22, src/detectors.ts):
-// sql/config-scope findings never set `stageId` at all, so `Finding.stageId`
-// is now `number | null | undefined`, and this `catalog` param is usually a
-// live `Finding[]`. `stageId == null` below already treats both the same.
+// Cross-widget stage recurrence: how many distinct board widgets (skew+straggler on one stage
+// still count as one, task-skew) flag a stage. Computed once from the full catalog each widget
+// already receives.
+// `stageId` may be number|null|undefined: sql/config-scope findings never set it; `stageId == null`
+// below treats both the same.
 export function stageWidgetFrequency(catalog                                                )                      {
   const perStage = new Map();
   for (const b of catalog) {
@@ -34,12 +30,9 @@ export function stageWidgetFrequency(catalog
   return freq;
 }
 
-// Canonical detector type → ALL-CAPS board tag vocabulary (AGENTS.md "Problem
-// flagging"). Single source of truth for every widget that needs to render a
-// catalog entry's tag outside its own dedicated board card, e.g. Bottleneck
-// Alerts, which lists findings across every detector type.
-// Exported (not just used internally by typeTag) so doc-sync checks can
-// enumerate every tag value without hand-duplicating this list elsewhere.
+// Canonical detector type -> ALL-CAPS board tag vocabulary. Single source of truth for every
+// widget that renders a catalog entry's tag outside its own card (e.g. Bottleneck Alerts).
+// Exported so doc-sync checks can enumerate every tag without hand-duplicating this list.
 export const TYPE_TAG_MAP                         = {
   skew: 'SKEW', shuffle: 'SHFL', spill: 'SPILL', gc: 'GC',
   coldStart: 'COLD', utilization: 'UTIL', memoryUtilization: 'MEM',
@@ -58,23 +51,34 @@ export function typeTag(type        )         {
   return TYPE_TAG_MAP[type] ?? type.toUpperCase();
 }
 
-// Human-readable tooltips for the terse spill-classification badges
-// (skew | vol | ?), surfaced via title= so the vocabulary is decipherable
-// without an external key. Kept domain-agnostic (Spark-internal terms only).
+// Human-readable tooltips for the terse spill-classification badges (skew | vol | ?), surfaced via
+// title=. Kept domain-agnostic (Spark-internal terms only).
 export const SPILL_CLASS_TITLE = {
   skew: 'Skew spill: a few heavy tasks spill while most do not; rebalance partitioning',
   volume: 'Volume spill: most tasks spill because data exceeds memory; add partitions',
   unclassified: 'Spill cause could not be classified',
 };
 
-/**
- * Worst (lowest IMPACT_BAND_ORDER) impact band across a list of findings,
- * `undefined` when the list is empty: shared by every widget that needs a
- * single band to badge/color a group of findings (Topbar, ShuffleIO,
- * GcPressure, TaskSkew, Failures, ConfigAudit, Alerts).
+// Short badge text, kept distinct from the long-form SPILL_CLASS_TITLE tooltip. Shared by
+// Spill.tsx's per-row badge and StageTable.tsx's spill-column cell.
+export const SPILL_CLASS_SHORT                                                 = {
+  skew: 'skew', volume: 'vol', unclassified: '?',
+};
+
+const DEFAULT_MAX_STAGE_IDS_SHOWN = 8;
+
+// Caps a stage-id list's printed length: a sql-scope finding's stageIds can run into the hundreds,
+// and joined raw that length balloons an auto-layout table's column width. Shared by FixTheseFirst
+// and PlanFindings.
+export function formatStageIdsLabel(stageIds          , maxShown         = DEFAULT_MAX_STAGE_IDS_SHOWN)         {
+  if (stageIds.length <= maxShown) return stageIds.join(', ');
+  return `${stageIds.slice(0, maxShown).join(', ')}, +${stageIds.length - maxShown} more`;
+}
+
+/** Worst (lowest IMPACT_BAND_ORDER) impact band across findings, undefined when empty. Shared by
+ * every widget that badges/colors a group of findings.
  * @param {{ impactBand: 'critical' | 'warning' | 'info' }[]} findings
- * @returns {'critical' | 'warning' | 'info' | undefined}
- */
+ * @returns {'critical' | 'warning' | 'info' | undefined} */
 export function worstImpactBand(
   findings                                                   ,
 )                                              {
@@ -89,12 +93,8 @@ export function escHtml(str         )         {
   return String(str).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
 }
 
-// Reduces an embedded HDFS/S3/absolute-path fragment in a string to its
-// basename (e.g. "Scan ExistingRDD ... - hdfs://host/a/b/c" -> "c"). Strings
-// with no such fragment pass through unchanged; this is NOT a general
-// truncator. Shared by cache-utilization.js (RDD names), detectors.js (plan
-// node names embedded in recommendation text), and stage-detail.js (plan
-// node detail labels).
+// Reduces an embedded HDFS/S3/absolute-path fragment to its basename (e.g. "... hdfs://host/a/b/c"
+// -> "c"). Strings with no such fragment pass through unchanged; NOT a general truncator.
 export function pathBasename(str         )         {
   const s = String(str);
   const m = s.match(/((?:hdfs?|s3[an]?):\/\/\S+|\/[^\s,)]+)/i);
@@ -103,10 +103,9 @@ export function pathBasename(str         )         {
   return segs[segs.length - 1] || s;
 }
 
-// "collect at /u02/hadoop/.../utils.py:1869" -> "collect at utils.py:1869". Splits
-// on the last " at " (Spark's own callsite format is "<operation> at <location>"),
-// then reduces the location to its basename via pathBasename (whose match already
-// includes a trailing ":<line>", since colons aren't excluded from its regex).
+// "collect at /u02/.../utils.py:1869" -> "collect at utils.py:1869". Splits on the last " at "
+// (Spark's callsite format), then reduces the location to its basename via pathBasename (whose
+// match keeps a trailing ":<line>").
 export function trimCallsite(callsite        )         {
   const idx = callsite.lastIndexOf(' at ');
   if (idx === -1) return callsite;
@@ -115,9 +114,8 @@ export function trimCallsite(callsite        )         {
   return `${operation} at ${pathBasename(location)}`;
 }
 
-// recent-files ids are `${name}::${size}::${lastModified}`: show just the
-// name. Preferred over `app.name`, which is usually identical across the two
-// runs being compared and so wouldn't distinguish baseline from candidate.
+// recent-files ids are `${name}::${size}::${lastModified}`: show just the name. Preferred over
+// app.name, which is usually identical across the two compared runs.
 export function runLabel(id        )         {
   return id.split('::')[0] || id;
 }
@@ -136,6 +134,82 @@ export function nsToMs(ns        )         {
   return ns / 1e6;
 }
 
+// Finding.value is number|string only because a couple of detectors (configAudit, stageFailed)
+// report a string; every other row formatter needs the numeric case and falls back to 0.
+export function numericValue(f                             )         {
+  return typeof f.value === 'number' ? f.value : 0;
+}
+
+// Low-level "number -> display string" step shared by every widget's per-finding label resolver:
+// each keeps its own type/metric routing and calls this only for the final value-to-string step,
+// so the same number renders identically ("42%", "2.3×") everywhere.
+export function formatMetricValue(unit                                                  , value        )         {
+  switch (unit) {
+    case 'pct': return `${value}%`;
+    // 'pct' expects an already-scaled 0-100 value (every detector's Finding.value convention); this
+    // variant is for the rare metric (slowHost's hostDurationShare) whose value is a raw 0-1 fraction.
+    case 'pctFraction': return `${Math.round(value * 100)}%`;
+    case 'ratio': return `${value}×`;
+    case 'ms': return formatDuration(value);
+    case 'count': return `${value}`;
+  }
+}
+
+// Finding.metric -> display unit, for the compact plan-graph finding chip. Keyed by the exact
+// `metric` string each per-stage/plan-advisor detector writes (see detectors.ts); a metric this map
+// doesn't cover, or a non-numeric `value` (stageFailed's failure reason, configAudit's config text),
+// yields no magnitude. 'bytes' handles the sub-KB tier formatBytes deliberately lacks; 'minutes'
+// converts to ms so it shares formatDuration's phrasing.
+const FINDING_METRIC_UNIT                                                                                         = {
+  shuffleReadBytes: 'bytes', memoryBytesSpilled: 'bytes', shuffleReadMax: 'bytes',
+  avgFileSizeBytes: 'bytes', smallerSideBytes: 'bytes', broadcastBytes: 'bytes',
+  speculationWasteMs: 'ms', retryWasteMs: 'ms', taskDurationP50: 'ms',
+  stageDurationMinutes: 'minutes',
+  'P95/median': 'ratio', 'max/median': 'ratio', pRatio: 'ratio', oiRatio: 'ratio',
+  taskStageSkew: 'ratio', hostMeanRatio: 'ratio', execMaxMedianRatio: 'ratio',
+  gcPct: 'pct', failureRate: 'pct', stragglerShare: 'pct',
+  hostDurationShare: 'pctFraction',
+  taskCount: 'count', speculativeTasks: 'count', subtreeOccurrences: 'count',
+};
+
+// formatBytes floors at KB (its guarded contract, see partition-sizing.test.tsx); the plan-graph
+// chip is the one place a sub-KB magnitude is meaningful (a smallFiles avgFileSizeBytes can be a
+// few hundred bytes), so render that tier here rather than widening the shared formatter.
+function formatChipBytes(bytes        )         {
+  if (bytes > 0 && bytes < 1000) return `${Math.round(bytes)} B`;
+  return formatBytes(bytes);
+}
+
+/** Compact magnitude for a finding's plan-graph chip (e.g. "4.2 GB", "3.2×", "45%"), taken from the
+ * finding's own `value` rendered in its `metric`'s unit. Returns null when `value` is non-numeric
+ * (stageFailed/configAudit reuse it for text) or the metric isn't in FINDING_METRIC_UNIT. */
+export function formatFindingMagnitude(finding                                              )                {
+  const unit = finding.metric ? FINDING_METRIC_UNIT[finding.metric] : undefined;
+  if (!unit || typeof finding.value !== 'number') return null;
+  switch (unit) {
+    case 'bytes': return formatChipBytes(finding.value);
+    case 'minutes': return formatDuration(finding.value * 60000);
+    case 'ms': return formatMetricValue('ms', finding.value);
+    case 'ratio': return formatMetricValue('ratio', finding.value);
+    case 'pct': return formatMetricValue('pct', finding.value);
+    case 'pctFraction': return formatMetricValue('pctFraction', finding.value);
+    case 'count': return formatMetricValue('count', finding.value);
+  }
+}
+
+/** The plan-graph finding chip's detail text: the finding's magnitude and, when a wall-clock
+ * recovery is claimed, "~<time>" (the optimistic `high` bound, matching formatImpactEstimateCompact),
+ * joined by " · " (e.g. "4.2 GB · ~38.0s"). null when neither part is available. */
+export function formatFindingChipDetail(
+  finding                                                                                                                     ,
+)                {
+  const magnitude = formatFindingMagnitude(finding);
+  const wallClock = finding.impactEstimate?.wallClock;
+  const recoverable = wallClock ? `~${formatDuration(wallClock.high)}` : null;
+  const parts = [magnitude, recoverable].filter(Boolean);
+  return parts.length > 0 ? parts.join(' · ') : null;
+}
+
 export function formatDuration(ms                    )         {
   if (!ms || ms <= 0) return '—';
   if (ms < 1000) return `${Math.round(ms)}ms`;

package/vendor-core/impact-band.js

@@ -1,11 +1,8 @@
                                                         
 import { STRAGGLER_FLOOR_PCT_WARN, STRAGGLER_FLOOR_PCT_CRIT } from './detectors.js';
 
-// Reused from `straggler`'s own thresholds (detectors.ts's
-// `floorPctWarn`/`floorPctCrit`, marked `NOT SOURCED: unvalidated` there
-// already): applying the same, already-accepted noise floor globally rather
-// than inventing a second, unrelated cutoff. Imported (not re-literaled) so
-// the two can't drift out of sync.
+// Reused from straggler's own thresholds (marked NOT SOURCED: unvalidated there): apply the same
+// accepted noise floor globally rather than inventing a second cutoff. Imported so they can't drift.
 const IMPACT_FLOOR_PCT_WARN = STRAGGLER_FLOOR_PCT_WARN;
 const IMPACT_FLOOR_PCT_CRIT = STRAGGLER_FLOOR_PCT_CRIT;
 
@@ -16,31 +13,28 @@ function appDurationMs(app                     )                {
 }
 
 /**
- * Overwrites `.impactBand` for any finding with a quantified wall-clock
- * estimate, grading it purely by recoverable time as a fraction of the
- * run's total duration: a full replace (can move a finding's fixed fallback
- * band in either direction once real recoverable time is known), not a
- * promote-only floor. Findings with no `wallClock` estimate
- * (`resourceOnly`/`informational` basis) or an unknown/zero app duration are
- * left exactly as the detector set them: there is no comparable figure to
- * grade them by, so the detector's own fixed classification stands as the
- * final value. Mutates in place and returns the same array, matching
- * `estimateImpact`'s own signature style.
+ * Overwrites `.impactBand` for any finding with a quantified wall-clock estimate, grading it purely
+ * by recoverable time as a fraction of the run's duration: a full replace (either direction), not a
+ * promote-only floor. Findings with no wallClock estimate (resourceOnly/informational), an
+ * unknown/zero app duration, or an explicit safety-signal exemption (partitionSizing's
+ * maxPartitionTooBig rule) are left as the detector set them. Mutates in place and returns the array.
  *
- * Grades `wallClock.high`, matching `triage-target.ts`/`FixTheseFirst.tsx`/
- * `formatImpactEstimateCompact`, which already rank and display severity by
- * the same optimistic figure. This is a deliberate split from
- * `view/impact-sort.ts`'s `'impact'` sort mode, which ranks by `.low`
- * instead: severity grading (this function) wants the best-case number a fix
- * could realistically claim, while the sort wants the guaranteed floor so an
- * optimistic-but-contended finding never outranks a smaller but certain one.
- * Same range, two different fields for two different questions — not an
- * inconsistency to reconcile.
+ * Grades wallClock.high (matching triage-target/FixTheseFirst, which rank by the same optimistic
+ * figure), a deliberate split from view/impact-sort.ts's 'impact' sort, which ranks by .low so an
+ * optimistic-but-contended finding never outranks a smaller certain one. Same range, two fields for
+ * two questions.
  */
 export function deriveImpactBand(findings           , app                     )            {
   const durationMs = appDurationMs(app);
   if (durationMs == null) return findings;
   for (const finding of findings) {
+    // maxPartitionTooBig is a hardcoded-critical OOM/crash-risk safety signal, not a
+    // time-recovery one, yet it carries a real wallClock estimate (unlike the other hardcoded-
+    // critical findings, which stay costOnly/informational and are exempted below by having no
+    // wallClock at all). Exempt it explicitly so a long-running job never demotes an active
+    // crash risk down to 'info' just because fixing it saves little wall-clock time relative to
+    // the run.
+    if (finding.type === 'partitionSizing' && finding.rule === 'maxPartitionTooBig') continue;
     const recoverableMs = finding.impactEstimate?.wallClock?.high;
     if (recoverableMs == null) continue;
     const pct = recoverableMs / durationMs;

package/vendor-core/impact-estimator.js

@@ -2,8 +2,7 @@
 import { computeOccupancy, estimateSingleStage, estimateMultiStage,                                              } from './occupancy.js';
 import { detectorCatalog } from './detectors.js';
 
-// Assumed shuffle-network throughput, ~1 Gbps. Starting assumption; tune against the
-// Task 10 real-log spot-check (docs/architecture.md's Impact estimation section).
+// Assumed shuffle-network throughput, ~1 Gbps. Starting assumption, unvalidated.
 const SHUFFLE_THROUGHPUT_BPS = 125_000_000;
 // Assumed disk I/O throughput for spilled data, ~200 MB/s (conservative HDD/SSD blend).
 const SPILL_IO_THROUGHPUT_BPS = 200_000_000;
@@ -22,11 +21,8 @@ const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
 // Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
 const RE_READ_THROUGHPUT_BPS = 125_000_000;
 
-// The stageSlowness detector (src/detectors.ts) flags a stage once its wall-clock
-// duration reaches `infoMin` minutes; that's the point beyond which the stage's time
-// stops being "normal", so it's also the floor for the waste this estimate reports.
-// Read from the detector's own catalog entry (not a duplicated literal) so the two
-// stay in sync automatically if the detector's threshold ever changes.
+// stageSlowness flags a stage at `infoMin` minutes; that's the floor for the waste this estimate
+// reports. Read from the detector's catalog entry so the two stay in sync automatically.
 const STAGE_SLOWNESS_THRESHOLD_MINUTES = (() => {
   const infoMin = detectorCatalog().find((d) => d.type === 'stageSlowness')?.thresholds?.infoMin;
   if (typeof infoMin !== 'number') {
@@ -35,9 +31,8 @@ const STAGE_SLOWNESS_THRESHOLD_MINUTES = (() => {
   return infoMin;
 })();
 
-// A finding with no quantifiable magnitude falls back to 'informational'; one that
-// still yields a rawWaste figure (independent of any stage window) falls back to
-// 'resourceOnly'. Never a fake {low: 0, high: 0}: wallClock is null in both cases.
+// No quantifiable magnitude -> 'informational'; a rawWaste figure with no stage window ->
+// 'resourceOnly'. Never a fake {low:0, high:0}: wallClock is null in both cases.
 function costOnly(estimateMethod                      , rawWaste                 )                 {
   return rawWaste
     ? { basis: 'resourceOnly', wallClock: null, estimateMethod, rawWaste }
@@ -67,9 +62,8 @@ function stageMappableWasteOrCostOnly(
   if (!stageIds || stageIds.length === 0) {
     return costOnly('modeled', rawWaste);
   }
-  // One waste event spread over a span of stages, not N independent wastes:
-  // apportion it evenly so estimateMultiStage's union cap doesn't have to
-  // absorb the same amount claimed once per stage.
+  // One waste event spread over a span of stages, not N independent wastes: apportion evenly so
+  // estimateMultiStage's union cap doesn't absorb the same amount claimed once per stage.
   const perStageWasteMs = wasteMs / stageIds.length;
   const wasteMsByStage = new Map(stageIds.map((id) => [id, perStageWasteMs]));
   const est = estimateMultiStage(stageIds, wasteMsByStage, stages                                          , occupancy);
@@ -77,13 +71,8 @@ function stageMappableWasteOrCostOnly(
   return { basis: est.basis, wallClock: est.wallClock, estimateMethod: 'modeled', rawWaste };
 }
 
-/**
- * Per-finding-type dispatch, populated incrementally: each formula task adds one
- * case here. A type with no case stays uncovered (no impactEstimate attached),
- * every type this design covers must eventually get one, per the Global Constraints'
- * "no silent omissions" rule; docs/architecture.md's Impact estimation table is the
- * authoritative completeness check, not this switch by itself.
- */
+/** Per-finding-type dispatch. A type with no case stays uncovered (no impactEstimate attached);
+ * docs/architecture.md's Impact estimation table is the authoritative completeness check, not this switch. */
 function computeEstimateForFinding(
   finding         ,
   stages                    ,
@@ -91,8 +80,7 @@ function computeEstimateForFinding(
 )                        {
   switch (finding.type) {
     case 'retryWaste': {
-      // The waste figure lives on the Stage, not the Finding: the detector only
-      // re-publishes it as `metric`/`value` (src/detectors.ts's retryWaste entry).
+      // The waste figure lives on the Stage, not the Finding: the detector only re-publishes it as metric/value.
       if (finding.stageId == null) return null;
       const stage = stages.get(finding.stageId);
       if (!stage) return null;
@@ -110,9 +98,8 @@ function computeEstimateForFinding(
       // The detector reports the gap as `metric: 'startupGapSeconds', value: <seconds>`.
       if (typeof finding.value !== 'number') return null;
       const wasteMs = finding.value * 1000;
-      // Time before any task starts can never overlap any stage; a genuine,
-      // unclipped point estimate, not tied to any stage's own gate (coldStart is
-      // app-scoped, stageId: null): see Non-goals in the original design spec.
+      // Time before any task starts can never overlap any stage; a genuine unclipped point estimate,
+      // not tied to any stage's gate (coldStart is app-scoped, stageId: null).
       return { basis: 'serial', wallClock: { low: wasteMs, high: wasteMs }, estimateMethod: 'measured' };
     }
     case 'gc': {
@@ -129,11 +116,8 @@ function computeEstimateForFinding(
         return costOnly('modeled', rawWaste);
       }
       const avgConcurrency = executorRunTime / stageDurationMs;
-      // jvmGCTime is a cross-task core-time sum, the same aggregation shape as
-      // executorRunTime; dividing by the stage's own average concurrency converts it
-      // to an approximate wall-clock figure. This rides on measured inputs but is a
-      // modeled approximation, not an exact reconstruction: hence 'modeled', and see
-      // docs/architecture.md's spot-check note next to this formula.
+      // jvmGCTime is a cross-task core-time sum (same shape as executorRunTime); dividing by the
+      // stage's average concurrency converts it to an approximate wall-clock figure. Modeled, not exact.
       const wasteMs = jvmGCTime / avgConcurrency;
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', rawWaste);
     }
@@ -149,13 +133,9 @@ function computeEstimateForFinding(
     }
     case 'straggler':
     case 'stageShape': {
-      // NOTE: this is a shared case for two finding types. straggler (which has
-      // no `rule` field to check) falls through to the max-P50 wall-clock
-      // computation below; every stageShape rule (including taskStageSkew) gets
-      // its own resourceOnly formula here, each returning early (not `break`,
-      // which would fall off the end of this function's switch and implicitly
-      // return `undefined` instead of `null` since the switch is the function's
-      // final statement).
+      // Shared case for two finding types. straggler (no `rule` field) falls through to the max-P50
+      // computation below; every stageShape rule returns early (not `break`, which would fall off
+      // the switch and return undefined instead of null since the switch is the function's last statement).
       if (finding.type === 'stageShape') {
         if (finding.rule === 'lowParallelism') {
           const stage = stages.get(finding.stageId          );
@@ -178,12 +158,9 @@ function computeEstimateForFinding(
           if (!stage) return null;
           const totalCores = (finding.totalCores                      ) ?? 0;
           const taskCount = stage.taskCount ?? 0;
-          // Cores idle during the straggler's tail, at achieved concurrency (not full
-          // cluster capacity, which is lowParallelism's own territory): satisfying this
-          // rule's trigger condition mathematically forces the occupancy-clipped
-          // wall-clock estimate to zero on every firing, so this is resourceOnly, not a
-          // wall-clock claim (see the Overlap caveat section in
-          // docs-site/contributor-guide/architecture/impact-estimation.md).
+          // Cores idle during the straggler's tail, at achieved concurrency (not full cluster
+          // capacity, which is lowParallelism's territory): this rule's trigger forces the
+          // occupancy-clipped estimate to zero on every firing, so it's resourceOnly, not a wall-clock claim.
           const idleCoreMs =
             Math.max(0, Math.min(totalCores, taskCount) - 1) *
             Math.max(0, (stage.taskDurationMax ?? 0) - (stage.taskDurationP50 ?? 0));
@@ -198,13 +175,10 @@ function computeEstimateForFinding(
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'measured', { value: wasteMs, unit: 'ms' });
     }
     case 'slowHost': {
-      // Three duration-based shapes, each carrying its absolute-ms figure under a
-      // different field (`value` is a ratio or a share in all three, never ms):
-      //   - the per-host mean branch, discriminated by `metric` (it sets no `variant`)
-      //   - the duration-share branch, discriminated by `variant`
-      //   - the multi-dimension branch, only for its taskTime dimension
-      // Every other shape (the byte-based multiDim dimensions) has no absolute
-      // figure today, so it stays informational.
+      // Three duration-based shapes, each carrying its absolute-ms figure under a different field
+      // (`value` is always a ratio/share, never ms): the per-host mean branch (discriminated by
+      // `metric`), the duration-share branch (`variant`), and the multiDim taskTime dimension. Every
+      // byte-based multiDim dimension has no absolute figure today, so it stays informational.
       const absoluteMs =
         finding.metric === 'hostMeanRatio' || finding.variant === 'durationShare'
           ? (finding.hostMeanMs                      )
@@ -223,13 +197,10 @@ function computeEstimateForFinding(
     case 'duplicatePlanSubtree': {
       const stageIds = finding.stageIds                        ;
       if (!stageIds || stageIds.length === 0) return null;
-      // The detector reports metric: 'subtreeOccurrences', value: <occurrences>, always
-      // >= 2 (its own `minOccurrences` threshold). Only the repeats past the first are
-      // redundant: computing the subtree once is real work, so the waste is
-      // (occurrences - 1) / occurrences of the contributing stages' time, not all of it.
+      // The detector reports subtreeOccurrences >= 2. Only repeats past the first are redundant:
+      // computing the subtree once is real work, so waste is (occurrences-1)/occurrences of the stages' time.
       const occurrences = typeof finding.value === 'number' ? finding.value : 0;
-      // Defensive only: the detector's own `minOccurrences` threshold guarantees occurrences
-      // >= 2 on real data, so this is a malformed-`value` fallback, not a real formula run.
+      // Defensive only: minOccurrences guarantees occurrences >= 2 on real data; a malformed-value fallback.
       if (occurrences < 2) return costOnly('none');
       const redundantFraction = (occurrences - 1) / occurrences;
       const wasteMsByStage = new Map                ();
@@ -261,8 +232,7 @@ function computeEstimateForFinding(
       if (!stage) return null;
       const diskBytesSpilled = stage.diskBytesSpilled ?? 0;
       const wasteMs = (diskBytesSpilled / SPILL_IO_THROUGHPUT_BPS) * 1000;
-      // Surfaces the number the formula actually uses: the finding's displayed `metric`
-      // is memoryBytesSpilled, but disk spill is what costs I/O time.
+      // Surfaces the number the formula uses: the displayed metric is memoryBytesSpilled, but disk spill costs the I/O time.
       return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: diskBytesSpilled, unit: 'bytes' });
     }
     case 'stageSlowness': {
@@ -288,12 +258,10 @@ function computeEstimateForFinding(
         const taskCount = stage.taskCount ?? 0;
         if (targetTaskCount > taskCount && taskCount > 0) {
           const stageDurationMs = Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
-          // Too few shuffle partitions means each task processes more bytes than the ideal
-          // target, doing serially what more partitions would let run concurrently: the waste
-          // is that serialized work, not the scheduling cost of the tasks you'd add to fix it
-          // (adding tasks INCURS overhead, it doesn't recover any). Model the achievable
-          // duration at target parallelism by scaling down proportionally to the partition
-          // shortfall, and claim the difference.
+          // Too few shuffle partitions means each task processes more than the ideal bytes,
+          // serializing work more partitions would run concurrently: the waste is that serialized
+          // work, not the scheduling cost of tasks you'd add (adding tasks incurs overhead, recovers
+          // nothing). Model the achievable duration at target parallelism by scaling down proportionally.
           wasteMs = stageDurationMs * (1 - taskCount / targetTaskCount);
         }
       } else {
@@ -330,8 +298,7 @@ function computeEstimateForFinding(
         return costOnly('measured', { value: finding.value, unit: 'mbSeconds' });
       }
       if (finding.variant === 'idleCores') {
-        // Idle core-time priced as memory held but unused: the same MB-seconds unit
-        // the wasteModel variant reports, so the two are comparable.
+        // Idle core-time priced as memory held but unused: the same MB-seconds unit as wasteModel, so comparable.
         const idleRateFraction = finding.idleRateFraction                      ;
         const allocatedMB = finding.allocatedMB                      ;
         const peakExecutors = finding.peakExecutors                      ;
@@ -342,9 +309,8 @@ function computeEstimateForFinding(
         }
         return costOnly('modeled');
       }
-      // Only the over-provisioned band is a waste; the near-capacity band is an OOM-risk
-      // signal with no magnitude to report, and the dataUnavailable shape has no inputs
-      // at all: both stay informational.
+      // Only the over-provisioned band is a waste; the near-capacity band is an OOM-risk signal with
+      // no magnitude, and the dataUnavailable shape has no inputs: both stay informational.
       if (finding.variant === 'memoryBand' && finding.rule === 'heapOverProvisioned') {
         const allocatedBytes = finding.allocatedBytes                      ;
         const heap = finding.heap                      ;
@@ -398,11 +364,9 @@ function computeEstimateForFinding(
       const numPartitions = (finding.numPartitions                      ) ?? 0;
       const numUncachedPartitions = Math.max(0, numPartitions - numCachedPartitions);
       const cachedBytes = memorySize + diskSize;
-      // Extrapolate the never-cached partitions' size from the CACHED partitions' own
-      // average size (uncached/cached, not uncached/total: `numCachedPartitions` partitions
-      // produced `cachedBytes`, not all `numPartitions` of them). `diskSize` is added once
-      // more on its own: those bytes are already cached, but on disk rather than memory, so
-      // re-reading them still costs I/O the way a genuinely-uncached partition would.
+      // Extrapolate never-cached partitions' size from the CACHED partitions' average (uncached/
+      // cached, not uncached/total: numCachedPartitions produced cachedBytes). diskSize is added
+      // once more: those bytes are cached but on disk, so re-reading them still costs I/O like an uncached partition.
       const uncachedBytes = numCachedPartitions > 0 ? (cachedBytes / numCachedPartitions) * numUncachedPartitions : 0;
       const uncachedOrSpilledBytes = uncachedBytes + diskSize;
       const wasteMs = (uncachedOrSpilledBytes / RE_READ_THROUGHPUT_BPS) * 1000;

package/vendor-core/ingest.js

@@ -1,11 +1,7 @@
-// Worker/CLI-shared message router: maps a worker (or Node-synchronous,
-// per src/cli/collect-run.js's dispatch()) message to its handler callback.
-// Exported so both contexts run one switch instead of hand-kept-in-sync
-// copies. `pendingTaskRequests` is optional: the CLI path never sends a
-// `getTaskData` request, so it never receives a `taskData` message back.
-// Handler dispatch is optional-chained (`handlers.onXxx?.(...)`), so a
-// caller that adds a new message type here without wiring its handler
-// fails silently (a no-op) rather than throwing.
+// Worker/CLI-shared message router: maps a worker (or Node-synchronous) message to its handler.
+// Exported so both contexts run one switch. `pendingTaskRequests` is optional: the CLI path never
+// sends a `getTaskData` request, so never receives `taskData` back. Handler dispatch is
+// optional-chained, so a caller that adds a message type without wiring its handler no-ops rather than throws.
 
                                  
                                        
@@ -77,11 +73,9 @@ export function createIngestClient()
 
   let handlers                 = {};
 
-  // The `new Worker(new URL(...), ...)` expression must appear inline, exactly
-  // like this, for Vite's worker plugin to statically detect and bundle it
-  // (including its own imports, e.g. vendor/fflate.js): routing the URL
-  // through an intermediate variable defeats that detection and leaves the
-  // worker's dependencies unbundled, 404ing at runtime in a production build.
+  // The `new Worker(new URL(...), ...)` expression must appear inline for Vite's worker plugin to
+  // statically detect and bundle it (with its own imports); routing the URL through a variable
+  // defeats detection and leaves the worker's deps unbundled, 404ing in a production build.
   function makeWorker() {
     worker = new Worker(new URL('./parser-worker.js', import.meta.url), { type: 'module' });
     worker.onmessage = ({ data }) => routeMessage(data, handlers, pendingTaskRequests);

package/vendor-core/job-groups.js

@@ -1,9 +1,6 @@
-// §8 Concurrent-job-group reliability guard (SparkLens JobOverlapAnalyzer).
-// Groups jobs by SQL execution id; jobs with none form their own singleton
-// group. If two DIFFERENT groups' [submissionTime, completionTime] intervals
-// overlap, wall-clock-based estimates (scaling simulator, efficiency model)
-// become unreliable. Jobs WITHIN a group overlapping (AQE/multi-stage) is
-// normal and not the signal.
+// §8 Concurrent-job-group reliability guard. Groups jobs by SQL
+// execution id (jobs with none form singleton groups). If two DIFFERENT groups' intervals overlap,
+// wall-clock-based estimates become unreliable. Jobs WITHIN a group overlapping (AQE) is normal.
 export function checkConcurrentJobGroups(jobs                                                                                                                             )                                                                        {
   const list = jobs ? [...jobs.values()] : [];
   // Build one interval per group: [min submission, max completion].