sparkforensics-server 0.3.0 → 0.5.0

package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js

@@ -1 +0,0 @@
-import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Widget rendering","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/widget-rendering.md","filePath":"contributor-guide/architecture/widget-rendering.md"}'),i={name:"contributor-guide/architecture/widget-rendering.md"};function n(c,e,r,s,l,g){return t(),d("div",null,[...e[0]||(e[0]=[a('<h1 id="widget-rendering" tabindex="-1">Widget rendering <a class="header-anchor" href="#widget-rendering" aria-label="Permalink to &quot;Widget rendering&quot;">​</a></h1><h2 id="widget-rendering-order-fixed-spec-§5" tabindex="-1">Render order (fixed, spec §5) <a class="header-anchor" href="#widget-rendering-order-fixed-spec-§5" aria-label="Permalink to &quot;Render order (fixed, spec §5) {#widget-rendering-order-fixed-spec-§5}&quot;">​</a></h2><p><code>src/view/Dashboard.tsx</code>&#39;s <code>FilteredBoard</code> renders inside a <code>&lt;main&gt;</code> that opens with <code>FindingFilterBar</code> and (only when an active filter empties both finding streams) <code>NoMatchBanner</code>, then a single <code>Scorecard</code> strip, then (when the active filter doesn&#39;t empty the board) a two-tab <code>Tabs</code> (<code>src/components/ui/tabs.tsx</code>, a base-ui primitive): <strong>Findings</strong> and <strong>Full app report</strong> (2026-09-03 tabbed-impact-band-board redesign, replacing the prior three stacked sections: All recommendations, Suggested Improvements, Full app report, with a merged, impact-grouped Findings tab and an always-reachable Full app report tab). Base-ui <code>Tabs</code> fully unmount the inactive <code>TabsContent</code> panel rather than hiding it: a widget mounted only in Findings (every routeable <code>REGISTRY</code> widget; see &quot;First investigation routing&quot; below) is not in the DOM at all while Full app report is active, and remounts fresh, with its own state reset, when the user switches back.</p><p><code>region</code> on <code>RegistryEntry</code> (<code>src/view/detector-registry.tsx</code>) is read again, but only to decide whether a widget always mounts: <code>isAlwaysMountedType()</code> flags the one <code>reference</code>-region type still carved out as an always-mounted exception, <code>coreLocality</code> → <code>CoreUsageArea</code>. The other <code>reference</code>-region types stay ordinary finding-gated instead: <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> are a product decision: a clean run on any of them isn&#39;t evidence worth surfacing unconditionally, so each collapses to a plain clean-check line like any other detector. The Findings tab (<code>src/view/widgets/ImpactBoard.tsx</code>) groups its content into three tiers, same as the retired Suggested Improvements section did, just re-sliced by impact band instead of living as one flat active grid. The three tiers: impact-banded rows and cards for every <code>REGISTRY</code> component and every recommendation row with at least one finding; a small always-visible grid holding just the one exception above (mounted unconditionally from <code>appModel</code> regardless of finding state, below the impact bands); and a collapsed &quot;Clean checks&quot; disclosure covering every remaining type with zero findings, built per detector <em>type</em> (<code>Object.keys(REGISTRY)</code>). Full app report stays structural-only (see &quot;ReferenceSection&quot; below).</p><p>Tags carry their own docs links; there is no separate legend widget. <code>TagBadge</code> (<code>src/view/ImpactBadge.tsx</code>) resolves its own tooltip. Its documentation anchor is the caller&#39;s <code>docAnchor</code> prop when that is a known anchor (call sites holding the finding pass <code>finding.docAnchor</code>; a widget header or grouped row passes <code>sharedDocAnchor(findings)</code>), else the type&#39;s single known anchor (<code>docAnchorForType</code>, <code>src/view/finding-tag-help.ts</code>). The prop matters for <code>configAudit</code>, whose four entries carry different anchors, so the type lookup finds none. With an anchor, the pill itself links into the docs panel, and (density <code>advanced</code> only) a second icon link opens that tag&#39;s entry in <code>docs-site/user-guide/understanding-findings.md</code> (<code>findingGuideUrl</code>, <code>packages/core/src/docs-site-config.ts</code>) in the same in-app docs panel; its <code>target=&quot;_blank&quot;</code> only applies to a modifier-click or when no <code>DocsProvider</code> is mounted. When a type has no vendor-doc anchor (e.g. <code>incompleteRun</code>), the pill links straight to that same guide entry instead of rendering as inert text, and the second icon link is skipped as redundant. Either way, a linked pill gets a visible dotted underline, not just a hover tooltip. <code>TagBadge</code>&#39;s <code>plainBadge</code> prop suppresses both links together; a Findings-tab recommendation row is the one place that never sets it (see below), since its badge cell isn&#39;t nested inside a button.</p><h3 id="findings-tab" tabindex="-1">Findings tab <a class="header-anchor" href="#findings-tab" aria-label="Permalink to &quot;Findings tab&quot;">​</a></h3><p>Rendered by <code>FilteredBoard</code>&#39;s <code>TabsContent value=&quot;findings&quot;</code> (<code>src/view/widgets/ImpactBoard.tsx</code>). Merges what used to be two stacked sections, All recommendations and Suggested Improvements, into one impact-ranked board. The recommendation-rollup logic lives in <code>src/view/widgets/FixTheseFirst.tsx</code> (kept as its own file and its own directly-testable exports, but no longer rendered as a standalone page section by <code>Dashboard.tsx</code>) and the active-widget logic lives in <code>src/view/widgets/Alerts.tsx</code> (same: kept, no longer rendered standalone). <code>ImpactBoard</code> calls <code>FixTheseFirst.tsx</code>&#39;s exported <code>useFixTheseFirstData</code> (eligible findings, rollup groups, the top triage target) and <code>Alerts.tsx</code>&#39;s exported <code>computeActiveWidgets</code> (the ranked active-<code>REGISTRY</code> list) to get the exact same data these two retired sections used to render independently, then regroups both by impact band via <code>groupImpactBand</code> (<code>FixTheseFirst.tsx</code>: a rollup group&#39;s representative member&#39;s impact band, the same finding whose impact band its own badge already shows) and each active widget&#39;s own <code>worstImpactBand</code>.</p><p>Eligible findings for the rollup are <code>catalog</code> ∪ <code>configFindings</code>, filtered to a <code>REGISTRY</code>-mapped type, with <code>incompleteRun</code> (a pipeline-completeness caveat, not an addressable fix; see the spec&#39;s <code>fixEffort</code> table) and <code>memoryUtilization</code>&#39;s <code>memoryBand</code>/<code>dataUnavailable</code> variant (a missing-evidence caveat already covered by Evidence availability&#39;s own <code>executorMetrics</code> entry, <code>packages/core/src/evidence-availability.ts</code>) both explicitly excluded. Findings are grouped strictly by <code>finding.type</code> via <code>buildRecommendationRollup</code> (<code>packages/core/src/recommendation-rollup.ts</code>, further split within a type by impact kind and, for <code>resource</code>, unit: never merged across types); each resulting group becomes one row: a type with exactly one finding renders that finding directly, a type with more than one collapses into a summary row. Within an impact band, group order comes from <code>buildRecommendationRollup</code>&#39;s own sort: <code>time</code> groups (a real <code>wallClock</code> claim) first, ranked among themselves by their union-capped <code>recoverableMsHigh</code> descending; then <code>resource</code> groups (<code>rawWaste</code> but no <code>wallClock</code>); then <code>count</code> groups (neither), both of the latter two ranked by worst impact band, never by their incomparable raw magnitudes. A summary row&#39;s tag and dot, plus its text, all come from the group&#39;s own highest-impact member (via the same three-tier comparator, computed locally in <code>FixTheseFirst.tsx</code>); the trailing stat depends on the group&#39;s kind (<code>×N · &lt;time&gt; recoverable</code>, <code>×N · resource-cost projection</code>, or a plain per-impact-band tally). Clicking it expands straight to the group&#39;s full, impact-ranked list, with no intermediate &quot;worst-K&quot; step, paginated at 10 rows per page (<code>data-testid=&quot;fix-these-first-group-row&quot;</code>; no pager renders for a group of 10 or fewer findings; it appears once a group exceeds 10). A band&#39;s rollup rows render as a headerless three-column <code>Table</code> (<code>src/components/ui/table.tsx</code>): every individual row, whether shown directly or inside an expanded group, is a <code>TableRow</code> (<code>data-testid=&quot;fix-these-first-row&quot;</code>, <code>data-finding-type</code>) with three <code>TableCell</code>s: the impact dot + ALL-CAPS tag as a real <code>TagBadge</code> (not <code>plainBadge</code>: nothing wraps it, so its own docs links stay real <code>&lt;a&gt;</code>s, same as everywhere else on the board); a text block inside its own nested <code>&lt;button&gt;</code> (a short imperative action label, e.g. &quot;Reduce shuffle size&quot;, from <code>findingActionLabel</code> (<code>src/view/finding-action-label.ts</code>), over the finding&#39;s own full <code>recommendation</code> sentence in smaller muted text, both wrapping rather than truncating); and a right-aligned monospace stage reference + impact figure (e.g. <code>St.49 · 20.1s</code>, via <code>ImpactEstimate.tsx</code>&#39;s shared <code>formatWallClockRange</code>/<code>formatRawWaste</code>). That inner button, not the row, is the click target: it routes via <code>selectTriageTargetForFinding</code> (<code>src/view/triage-target.ts</code>), the same per-finding resolver Stage Summary Table&#39;s own control uses (see &quot;First investigation routing&quot; below); a <code>TypeGroupRow</code>&#39;s own inner button toggles its expand state instead (<code>aria-expanded</code>) and its expanded findings render as further <code>TableRow</code>s indented one badge-cell notch to read as the group&#39;s sub-list.</p><p>Each impact band (<code>ImpactBoard.tsx</code>&#39;s own <code>ImpactGroup</code>, one call per entry of <code>IMPACT_BAND_ORDER_LIST = [&#39;critical&#39;, &#39;warning&#39;, &#39;info&#39;]</code>) is a <code>&lt;section aria-label=&quot;Critical&quot; | &quot;Warning&quot; | &quot;Info&quot;&gt;</code> with an <code>&lt;h3&gt;</code> heading, and renders nothing (not even the heading) when it has neither a rollup row nor an active widget: a run with no critical findings has no &quot;Critical&quot; heading or section at all. Inside a band, rollup rows render first as the headerless <code>Table</code> described above, followed by that band&#39;s active <code>REGISTRY</code> widget cards (<code>computeActiveWidgets</code>&#39;s ranked list, filtered to this impact band) in their own <code>WidgetGrid</code>: every one of <code>orderedWidgets()</code>&#39;s deduped <code>REGISTRY</code> components <em>except</em> the one always-mounted one below, with at least one finding in <code>catalog</code> ∪ <code>configFindings</code>. Within a band, active widgets keep <code>orderedWidgets()</code>&#39;s own <code>action</code>-region-first, ascending-<code>DETECTORS</code>-order tiebreak. <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> are <code>reference</code>-region types but aren&#39;t always-mounted exceptions, so a Cache Storage, Memory Utilization, or Executor Utilization card with an active finding surfaces in its own impact band like any other active widget.</p><p>Below the impact bands, <code>Alerts.tsx</code>&#39;s exported <code>AlwaysVisibleAndCleanChecks</code> (shared verbatim with the retired standalone <code>Alerts</code> component) renders the same two tiers it always did, now living outside the impact-band grouping entirely rather than as this section&#39;s second and third tier: a small always-visible grid holding just Core Usage by Locality (<code>coreLocality</code>, resolving to <code>CoreUsageArea</code>), mounted unconditionally from <code>appModel</code> regardless of finding state (<code>alwaysMountedWidgets()</code>/<code>isAlwaysMountedType()</code>), carrying its own impact-band indicator when a finding is active instead of collapsing to a clean-check line on a clean run; and a collapsed &quot;Clean checks&quot; disclosure of <code>CleanCheckRow</code> lines (<code>src/view/widgets/CleanCheckRow.tsx</code>: label, the threshold it was measured against via <code>getThresholdSummary</code>, and &quot;No fix needed.&quot;) built per detector <em>type</em> (every <code>REGISTRY</code> key except that one always-mounted key): a clean run lands <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> here too, same as any ordinary action-region type. Caching Opportunities, Config Audit, and the four split Plan Advisor widgets (Redundant Plan Subtree, Excessive Small Files, Missed Broadcast Join, Oversized Broadcast Join) render through the ordinary active/clean paths above (see <a href="./board-widgets.html#board-widgets-beyond-the-fixed-six">Board widgets beyond the fixed six</a>). One consequence of this always-visible grid sitting below every impact band: a <code>critical</code>-band <code>coreLocality</code> finding still renders in that lower grid, below the <code>info</code>-band widgets above it, a deliberate tradeoff the spec accepted in exchange for never losing the widget on a clean run, not a ranking bug.</p><h3 id="per-widget-list-sort-mode" tabindex="-1">Per-widget list sort mode <a class="header-anchor" href="#per-widget-list-sort-mode" aria-label="Permalink to &quot;Per-widget list sort mode&quot;">​</a></h3><p><code>src/view/impact-sort.ts</code> (<code>sumWallClockLow</code>, <code>hasSortableImpact</code>, <code>byImpactDesc</code>, <code>stageIdOf</code>, <code>minStageId</code>, <code>byStageAsc</code>) and <code>src/view/SortModeToggle.tsx</code> are a shared, opt-in pair a Findings-tab active widget can use to let its own expanded, multi-row list default to potential-savings order (<code>wallClock.low</code> descending, the guaranteed-floor bound, not the optimistic <code>high</code>) instead of stage number. Each of the widgets below holds local <code>sortMode</code> state (<code>SortMode</code>, <code>&#39;impact&#39; | &#39;stage&#39;</code>, default <code>&#39;impact&#39;</code>) and renders a <code>SortModeToggle</code> in the <code>WidgetCard</code> <code>badges</code> slot, so it sits on the title row itself rather than floating in the body: a button alongside the widget&#39;s tag badges (a <code>WidgetCard</code> header renders <code>badges</code> beside the title heading, never inside another control). Every one of the former combined widgets&#39; split single-type widgets carries the same <code>canToggleSort</code> check its parent did, so the toggle only actually renders for a type whose finding can carry a wall-clock estimate: <code>Skew.tsx</code>, <code>TinyTask.tsx</code> (<code>stageShape</code>&#39;s own rules never produce one, so <code>StageShape.tsx</code> carries the same check but it never fires), <code>ShuffleIO.tsx</code> (narrowed to <code>shuffle</code>), <code>PartitionSizing.tsx</code>, <code>Spill.tsx</code>, <code>GcPressure.tsx</code> (both its high-GC and low-GC sections, one shared toggle), <code>RetryWaste.tsx</code> (its siblings <code>StageFailed.tsx</code>/<code>TaskFailures.tsx</code> carry the same check, but <code>stageFailed</code>/<code>failures</code> are <code>estimateMethod: &#39;none&#39;</code>, so it never fires there either), <code>SlowHost.tsx</code>, <code>StageSlowness.tsx</code>, <code>Straggler.tsx</code>, <code>SpeculationWaste.tsx</code>, <code>ColdStart.tsx</code> (five widgets now, one per type, each sorting only its own flat issue list; the old cross-type &quot;coldStart sinks to the bottom under By stage&quot; note no longer applies now that each type has its own single-type list), and <code>DuplicatePlanSubtree.tsx</code>/ <code>SmallFiles.tsx</code>/<code>UnderBroadcast.tsx</code>/<code>OverBroadcast.tsx</code> (each reorders its own list by <code>stageIdOf</code>, the lowest stage id its finding touches; the old cross-type <code>minStageId</code> group-ordering tiebreak no longer applies, since each type is its own widget now). Its <code>alternateOrderLabel</code> prop is required and every caller passes &quot;By stage&quot;: stage number is the one axis every one of these widgets&#39; items can be compared on, unlike the impact/raw-metric order this pattern replaced (dropped: comparing findings by impact band ranks them by how bad they are, not by how much fixing them would save, which is a worse default now that a real potential-savings figure exists to sort by instead). A per-stage detector&#39;s own <code>finding.stageId</code> is the sort key directly; a sql-scope finding that spans several stages (<code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>underBroadcast</code>, <code>overBroadcast</code>, via <code>stageIds</code>) sorts by the lowest stage id it touches (<code>stageIdOf</code>). The toggle only renders when <code>hasSortableImpact</code> finds at least one wall-clock claim in the list; re-sorting a list with none would be a silent no-op. Both comparators return <code>0</code> when neither side has a comparable value, so those items keep their prior relative order (<code>Array.prototype.sort</code>&#39;s stability) rather than being shuffled. <code>FixTheseFirst</code>&#39;s own expanded group list (see &quot;Findings tab&quot; above) already sorted by impact before this pattern existed and does not use it; it has no stage-order toggle.</p><h3 id="gold-standard-row-expand-contract" tabindex="-1">Gold Standard row/expand contract <a class="header-anchor" href="#gold-standard-row-expand-contract" aria-label="Permalink to &quot;Gold Standard row/expand contract&quot;">​</a></h3><p>Every <code>REGISTRY</code> widget renders its content as N≥1 rows, using <code>Skew.tsx</code> as the reference implementation. A widget&#39;s data shape (chart, table, single app-scoped scalar) is never by itself a reason to skip this: the three named exceptions below are the only ones. Any further exception must get its own entry here, justified in the same PR that introduces it.</p><ul><li>Tier A (universal): <code>WidgetCard</code> chrome, an impact-band left-border + dot, ALL-CAPS <code>TagBadge</code>s, a <code>SortModeToggle</code> when <code>hasSortableImpact</code> is true, a <code>finding-anchor</code> ref on any focusable/deep-linkable unit, and an early <code>null</code> return on an empty catalog filter (except Core Usage by Locality, the one always-mounted widget that renders unconditionally from <code>appModel</code> via <code>alwaysMountedWidgets()</code>/<code>isAlwaysMountedType()</code> rather than early-returning on an empty catalog filter).</li><li>Tier B (the row/expand pattern): a row&#39;s collapsed state shows its core metric(s), <code>ImpactEstimate</code>, its recommendation text, any config-hint code snippet/list, and docs links, all unconditionally. Confidence and evidence are unconditional too, since the 2026-09 redesign: <code>RowStatusCluster</code> (<code>src/view/RowStatusCluster.tsx</code>) is a single, fixed-position pill combining both, replacing the old per-row <code>ExpandToggleButton</code> + <code>ConfidenceMarker</code> + <code>EvidenceLink</code> trio entirely. Every widget that carries one wraps it in <code>AdvancedOnly</code> (<code>src/view/AdvancedOnly.tsx</code>), so it only renders at the Advanced density tier: this is a content-visibility gate (Basic vs. Advanced), not a per-row collapse, and it&#39;s applied consistently across every adopting widget. There is nothing left to gate behind a per-row click for confidence or evidence in any widget. A widget-header <code>RowStatusCluster</code> (passed as <code>WidgetCard</code>&#39;s <code>statusBadge</code> prop) carries one further, unrelated gate on top of <code>AdvancedOnly</code>: <code>open &amp;&amp; statusBadge</code> (<code>WidgetCard.tsx:138,141</code>) keeps it out of the collapsed header, matching the comment on <code>statusBadge</code> itself (&quot;kept separate so tag/impact badges stay visible in the collapsed summary while a <code>RowStatusCluster</code>-style ... marker doesn&#39;t&quot;). This is the card&#39;s own expand/collapse, not a separate reveal-on-click built for the cluster, and it composes with density as an AND: a header cluster needs both Advanced tier and an expanded card. A row with neither renders no cluster at all (<code>RowStatusCluster</code> returns <code>null</code>). Confidence renders as plain, non-interactive text (&quot;<code>{confidence} confidence</code>&quot;, no further suffix) with a <code>title</code> + <code>aria-describedby</code> tooltip carrying the full <code>validationRequired</code> detail: never itself a click target. Evidence renders as the cluster&#39;s one real click target: a button whose visible text is just the evidence label (e.g. &quot;SQL plan&quot;) but whose accessible name always carries the &quot;Evidence: ...&quot; prefix via an explicit <code>aria-label</code>, regardless of what else sits nearby (the old <code>bare</code> prop/prefix distinction is gone: there&#39;s only one form now). Clicking it calls the same <code>revealEvidence</code> navigation the retired <code>EvidenceLink</code> used. <code>RowStatusCluster</code>&#39;s fixed slot is the row&#39;s own stage-pill/title line (<code>justify-between</code>, cluster right-aligned), the same position in every adopting widget, not floating with the recommendation text below. Lists longer than <code>VISIBLE_LIMIT</code> (6, <code>packages/core/src/format-utils.ts</code>) still get page-based navigation (Previous/Next, <code>usePagedRows</code>/<code>RowPagination</code>, <code>src/view/usePagedRows.ts</code>/<code>src/view/RowPagination.tsx</code>) instead of rendering unconditionally: the one sanctioned cap mechanism across every Tier B widget. A Tier B widget whose rows are also triage-routable (wires <code>useFindingAnchor</code>) must pass its <code>routeIndex</code> (the routed finding&#39;s position in the paginated list) into <code>usePagedRows</code>, or a deep-linked route to a finding beyond the first page silently degrades to focusing the widget&#39;s disclosure title instead of the row.</li></ul><p>The following widgets carry a <code>RowStatusCluster</code>, all Advanced-only: <code>Spill.tsx</code> (confidence only), <code>ConfigAudit.tsx</code>&#39;s widget-header control (evidence only, a widget-wide constant rather than per-finding data, since the evidence key doesn&#39;t vary per row; also the control shown in the widget&#39;s empty-findings/no-data states), <code>MemoryUtilization.tsx</code>&#39;s rows (confidence + evidence together; <code>ExecutorUtilization.tsx</code>&#39;s rows, split out of the same former combined widget, carry neither), each of the four split Plan Advisor widgets&#39; (<code>DuplicatePlanSubtree.tsx</code>, <code>SmallFiles.tsx</code>, <code>UnderBroadcast.tsx</code>, <code>OverBroadcast.tsx</code>) per-row control (evidence only) plus their own widget-header marker (confidence only, one marker per widget now that each is its own finding-type, replacing the old <code>PlanFindings.tsx</code> per-group heading), <code>CoreUsageArea.tsx</code>, <code>EfficiencyModel.tsx</code>, <code>PlanView.tsx</code> (three call sites), <code>ScalingSim.tsx</code> (two call sites), and <code>WastedCoreHours.tsx</code> (all confidence-only, standalone rather than per-finding-row: a card-header badge, a widget-level single marker, a plan-tree node/summary-row marker, or an &quot;unavailable data&quot; message: the same component, same visual language, regardless of where it sits); and, since <code>skew</code>/<code>straggler</code>/<code>gc</code> started disclosing their own unvalidated noise-floor thresholds (confidence only, per-row, no <code>evidenceKey</code> passed), <code>GcPressure.tsx</code>&#39;s rows, <code>Straggler.tsx</code>&#39;s rows, <code>CachingOpportunity.tsx</code>&#39;s rows (the <code>cachingOpportunity</code> detector scales <code>confidence</code> per finding via <code>cachingReuseConfidence</code>, so the badge sits next to each row&#39;s recommendation rather than as a single caveat below the table), and the shared <code>StageFindingGroup.tsx</code> row (adopted by <code>Skew.tsx</code>/<code>StageShape.tsx</code>/<code>TinyTask.tsx</code>, though today only <code>skew</code> findings actually carry a <code>confidence</code> field).</p><p>One named exception:</p><ul><li><code>Skew.tsx</code>/<code>StageShape.tsx</code>/<code>TinyTask.tsx</code>: their per-row histogram toggle (<code>ExpandToggleButton</code> + <code>useExpandableRow</code>) gates a lazily-fetched duration histogram, unrelated to confidence/evidence, so it was untouched by the 2026-09 redesign and untouched again when <code>RowStatusCluster</code> was later added alongside it in the shared <code>StageFindingGroup.tsx</code> row. The two controls coexist per row. <code>ExpandToggleButton</code> is now single-purpose (always the task-detail toggle) since these three (split out of the former combined <code>TaskSkew.tsx</code>) are its only remaining adopters.</li></ul><p>No toggle at all, unconditional recommendation/content, same as before this redesign (unaffected either way, since these widgets never carried confidence/evidence display in the first place): <code>IncompleteRun.tsx</code>, <code>ShuffleIO.tsx</code>, <code>PartitionSizing.tsx</code>, <code>StageFailed.tsx</code>, <code>TaskFailures.tsx</code>, <code>RetryWaste.tsx</code>, <code>ColdStart.tsx</code>, <code>SlowHost.tsx</code>, <code>StageSlowness.tsx</code>, <code>SpeculationWaste.tsx</code>, <code>ExecutorCountChart.tsx</code>, <code>ExecutorUtilization.tsx</code>, <code>JobFailures.tsx</code>, <code>CacheUtilization.tsx</code>, <code>AutoscalingChurn.tsx</code>, and the four split Plan Advisor widgets (<code>DuplicatePlanSubtree.tsx</code>, <code>SmallFiles.tsx</code>, <code>UnderBroadcast.tsx</code>, <code>OverBroadcast.tsx</code>), split out of, respectively, the former combined <code>ExecutorTimeline.tsx</code>, <code>Failures.tsx</code>, and <code>PlanFindings.tsx</code>, none of which carried a per-row toggle either.</p><p>Three named exceptions to specific pieces of the contract, not to the row wrapper or pagination, which still apply to all three:</p><ul><li><code>CoreUsageArea.tsx</code>&#39;s non-local-stage list is a derived stat breakdown (<code>computeCoreLocalityRatio</code>&#39;s <code>topStages</code>), not a <code>Finding[]</code>: there is no per-row finding or recommendation to reveal. Paginated like every other Tier B list, but with no per-row expand.</li><li><code>CacheUtilization.tsx</code>&#39;s RDD <code>&lt;Table&gt;</code> holds reference columns (name, storage level, partitions, memory, disk bytes) with no recommendation attached to any row. Paginated like every other Tier B list, but with no per-row expand. Its separate recommendation list below the table is full Tier B (unconditional, no toggle, per the paragraph above).</li><li><code>IncompleteRun.tsx</code> leads with no metric of its own: this app-scoped, single-finding widget has no natural lead figure distinct from its title/badge, unlike every other Gold Standard widget, which leads with a percentage, duration, or count.</li></ul><h3 id="referencesection" tabindex="-1">ReferenceSection <a class="header-anchor" href="#referencesection" aria-label="Permalink to &quot;ReferenceSection&quot;">​</a></h3><p>Rendered by <code>FilteredBoard</code>&#39;s <code>TabsContent value=&quot;full-report&quot;</code>, tab label &quot;Full app report&quot;. A plain <code>&lt;div&gt;</code>, structural-only, reading none of <code>REGISTRY</code>/<code>orderedWidgets()</code> at all. Its heading is a visually-hidden (<code>sr-only</code>) <code>&lt;h2&gt;Full app report&lt;/h2&gt;</code>, matching the tab&#39;s own label for the accessibility-tree heading outline without visually duplicating the tab text. No longer an <code>Accordion</code>: base-ui <code>Tabs</code> unmount this panel entirely while Findings is active (see &quot;Render order&quot; above), so there is no &quot;collapsed but present&quot; state left to model, and this content is simply not in the DOM until the user clicks the Full app report tab. Its exact order is WallClock → Timeline → Executor Count Over Time (<code>ExecutorCountChart.tsx</code>, the executor add/remove count chart extracted out of the former combined <code>ExecutorTimeline.tsx</code>; not driven by any finding, so it isn&#39;t a <code>REGISTRY</code> entry) → StageTable → a <code>WidgetGrid</code> holding Evidence availability → ETL Phase Attribution → What-If Executor Scaling → Compute Efficiency → Wasted Core-Hours → Core-Usage Distribution. Scorecard used to lead this section; it now renders once, above the tabs themselves, in <code>FilteredBoard</code> (<code>src/view/Dashboard.tsx</code>), so it stays visible regardless of which tab is active rather than living inside either one (a three-tile run-info row: Wall-clock, Efficiency, Wastage; see <a href="./board-widgets.html#board-widgets-beyond-the-fixed-six">Board widgets beyond the fixed six</a>). WallClock, Timeline, StageTable and every tile in the grid beside them (Evidence availability included) all render immediately and fully expanded. No detector-driven <code>REGISTRY</code> card renders in this section any more: Memory Utilization, Executor Utilization, Core Usage by Locality, and Cache Storage all moved to the Findings tab above (Cache Storage, Memory Utilization, and Executor Utilization only surface there when they have an active finding; Core Usage by Locality alone is always-mounted).</p><p>The Evidence availability card is the persistent, non-impact-band ledger <a href="./worker-protocol.html#evidence-availability-contract-v1">defined in the worker protocol</a>, not an alert or detector widget. An <code>Evidence: …</code> control appears only where a conclusion or unavailable report lens declares a relevant ledger dependency. <code>revealEvidence</code> (<code>src/view/EvidenceAvailabilityContext.tsx</code>) sets <code>referenceOpen</code> true, which <code>DashboardContent</code> (<code>src/view/Dashboard.tsx</code>) watches in a <code>useEffect</code> and translates into <code>setActiveTab(&#39;full-report&#39;)</code>, replacing what used to be &quot;expand the Reference accordion&quot;: mouse and keyboard activation switches to the Full app report tab, opens the ledger card, then focuses the referenced stable entry id (<code>evidence-availability-&lt;key&gt;</code>). The control explains evidence availability; it does not promise an unavailable signal would have produced a finding.</p><p><code>WidgetGrid</code> (both inside a Findings impact band and inside <code>ReferenceSection</code>): collapsed cards occupy one responsive column; opening a card expands it across the full row. Nested <code>WidgetCard</code>s are isolated from the parent grid state. A Findings-tab <code>FixTheseFirst</code> rollup row is a table row, not a <code>WidgetGrid</code> card, and carries no collapse/expand state of its own (a <code>TypeGroupRow</code> summary row does own its own local expand/pagination state). Full app report&#39;s non-<code>REGISTRY</code> tiles (ETL Phase Attribution, What-If Executor Scaling, Compute Efficiency, Wasted Core-Hours, Core-Usage Distribution) sit in their own <code>WidgetGrid</code> alongside Evidence availability, but that grid has no <code>widgetId</code> wired to any card (see &quot;First investigation routing&quot; below), so nothing there is ever a route destination.</p><p>Firm constraint: <code>orderedWidgets()</code> (in <code>src/view/detector-registry.tsx</code>) still runs the single <code>DETECTORS</code>-ascending sort exactly as <code>dashboard-renderer.js</code> used to, and still sorts <code>action</code> components ahead of <code>reference</code> ones within that one list; the component-identity dedup it used to need is gone now that every <code>REGISTRY</code> entry maps to its own unique component (2026-09 widget/finding-type 1:1 mapping redesign). <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> (<code>reference</code>-region) can still reach the active grid alongside <code>duplicatePlanSubtree</code> (<code>action</code>-region) and any other active finding, since none of them is the one always-mounted exception filtered out before that grid. <code>computeActiveWidgets</code> (<code>Alerts.tsx</code>) is still its main consumer, now filtered through <code>isAlwaysMountedType()</code> to exclude that one always-mounted component; the clean-check list bypasses <code>orderedWidgets()</code> entirely, iterating <code>Object.keys(REGISTRY)</code> per type instead (see &quot;Findings tab&quot; above). <code>DETECTORS</code>&#39; own array order and iteration, plus its cross-detector <code>suppressWhen</code> logic (e.g. <code>stageSlowness</code> deferring to <code>slowHost</code>, see <a href="./detector-contract.html#detector-contract">Detector contract</a>), live entirely in <code>packages/core/src/detectors.ts</code>/<code>packages/core/src/analyzer.ts</code>, untouched by this redesign. Which React component each finding type resolves to is still registry-owned.</p><h3 id="first-investigation-routing" tabindex="-1">First investigation routing <a class="header-anchor" href="#first-investigation-routing" aria-label="Permalink to &quot;First investigation routing&quot;">​</a></h3><p>The dashboard has a view-only per-finding route, not a single first-investigation pick any more: every Findings-tab recommendation row (rendered by <code>FixTheseFirst.tsx</code>&#39;s row components inside <code>ImpactBoard</code>) and the visible-finding control in Stage Summary (<code>StageTable.tsx</code>) each independently resolve and request their own target. <code>detector-registry.tsx</code> owns each routeable finding type&#39;s stable <code>widgetId</code>, widget title, and finding label; identity and copy come from there, not from runtime position or raw detector type. Eligible targets have a mapped, routeable registry entry and a non-empty trimmed recommendation (<code>targetForFinding</code>, <code>src/view/triage-target.ts</code>). The target&#39;s widget still renders every affected stage in its local order.</p><p><code>src/view/triage-target.ts</code> also still exports <code>selectTriageTarget</code>, the whole-catalog &quot;single highest-potential-savings finding&quot; picker that used to back a start-here callout and each per-tag headline tile in the retired <code>ProblemHeadlines.tsx</code> (potential savings first via <code>impactEstimate.wallClock.high</code>, then <code>orderedWidgets()</code> widget order, then catalog order; impact band plays no part). Ranking by potential savings replaced the earlier severity-first routing once the occupancy-weighted impact estimator gave every finding a real, comparable <code>impactEstimate.wallClock</code> figure: severity-first could point the single &quot;start here&quot; callout at a <code>skew</code>/<code>straggler</code> finding ranked <code>critical</code> on a ratio basis while its occupancy-clipped recoverable time was near zero, passing over a lower-severity finding with an order-of-magnitude larger real recoverable-time estimate right next to it. Nothing in the app calls <code>selectTriageTarget</code> any more: <code>FixTheseFirst</code> already ranks every eligible finding by impact magnitude, so a separate single-winner pick has no call site left. <code>tests/view/triage-target.test.ts</code> still covers it directly.</p><p>The route is re-derived from the current catalog before each asynchronous step. Its identity is object-reference equality against the current catalog (<code>catalog.includes(finding)</code> in <code>selectTriageTargetForFinding</code>, <code>src/view/triage-target.ts</code>) rather than a derived key: a new parse yields new finding object references, so reference presence alone detects staleness. It is not persisted and does not extend the core <code>Finding</code> contract; cross-session consumers (exports, future URL-restored state) use the core <code>Finding.id</code> instead (see <a href="./worker-protocol.html#finding-identity">Finding identity</a>).</p><p><code>Dashboard</code> owns disclosure and navigation. Every routeable <code>REGISTRY</code> widget now lives in the Findings tab (no <code>REGISTRY</code> widget renders inside Full app report any more; see &quot;Render order&quot; above), so <code>requestRoute</code> (<code>DashboardContent</code>, <code>src/view/Dashboard.tsx</code>) starts with an unconditional <code>setActiveTab(&#39;findings&#39;)</code>, whether the request came from a control already on that tab or from Full app report&#39;s own Stage Summary route link. That tab switch can unmount and remount the whole Findings subtree in the same commit as the route landing (base-ui <code>Tabs</code> fully unmounts the inactive panel; see &quot;Render order&quot; above), which two routing paths have to account for: <code>reportWidgetOpen</code> no longer clears a route just because the target <code>WidgetGridItem</code> hasn&#39;t registered yet in this commit (a child <code>WidgetCard</code> reports its own open state before its parent <code>WidgetGridItem</code>&#39;s registration effect runs on a fresh mount, so treating &quot;not registered yet&quot; as &quot;stale&quot; would drop the route before the parent even gets a chance to register it); and a paginated widget that jumps its own page to reveal a routed row (<code>GcPressure.tsx</code>, <code>StageFailed.tsx</code>, <code>TaskFailures.tsx</code>, <code>RetryWaste.tsx</code>) computes that jump during render, not only in a post-commit effect, so the row exists in the DOM in time for this same commit&#39;s anchor lookup instead of one commit later.</p><p>After the target card commits open, its wrapper scrolls with a top margin below the fixed header and its disclosure button receives visible, temporary route focus (using instant scrolling for reduced motion). Requests use a latest-request-wins token (<code>requestTokenRef</code>) that guards every step and revalidates; a stale, missing, unmounted, changed-catalog, or changed-active-file target cancels without redirecting, scrolling, or moving focus. Widget registration probes survive StrictMode double-mounts via an <code>unregistrationCheck</code> that re-validates before clearing.</p><p>This route deliberately does not include Config Audit, detector/parser or threshold changes, Stage Detail routing, or generic badges, tags, chips, or StagePills as controls.</p><h3 id="widget-placement" tabindex="-1">Widget placement <a class="header-anchor" href="#widget-placement" aria-label="Permalink to &quot;Widget placement&quot;">​</a></h3><p>The Plan Explorer (<code>src/view/widgets/PlanExplorer.tsx</code>) is embedded inside a flagged stage&#39;s own row, not as a standalone widget. Neither Task Skew nor Spill embed it any more: both dropped their own <code>PlanExplorer</code> trigger entirely (not merely re-gated behind a toggle), since the same stage&#39;s plan is already reachable, unconditionally, from <code>StageDetailDialog</code>. Shuffle I/O is <code>PlanExplorer</code>&#39;s only remaining call site: it has no row-expansion toggle of its own, so its trigger renders as a direct sibling in the row body.</p><p>Plan detection logic belongs in a <code>packages/core/src/detectors.ts</code> entry (<code>scope:&#39;sql&#39;</code>) consuming <code>planTree</code>. <code>packages/core/src/plan-summary.ts</code> is display-only summarization; do not add detection heuristics there.</p><p><code>StageHeader.tsx</code> (<code>StagePill</code> plus a <code>&lt;span&gt;</code> naming the stage) vs. bare <code>StagePill</code>/<code>StagePillGroup</code> is a deliberate row-density choice, not an inconsistency: <code>StageHeader</code> is for single-stage-per-row detail widgets where naming the stage adds context (<code>SlowHost.tsx</code>, <code>StageSlowness.tsx</code>, <code>Straggler.tsx</code>, <code>SpeculationWaste.tsx</code>, <code>GcPressure.tsx</code>, <code>Skew.tsx</code>, <code>StageShape.tsx</code>, <code>TinyTask.tsx</code>, <code>PartitionSizing.tsx</code>), while bare <code>StagePill</code>/<code>StagePillGroup</code> is for compact, multi-row lists where many stages appear per widget (<code>StageFailed.tsx</code>, <code>TaskFailures.tsx</code>, <code>RetryWaste.tsx</code>, the four split Plan Advisor widgets, <code>Spill.tsx</code>, and <code>StageTable.tsx</code>). <code>ShuffleIO.tsx</code> uses both in different parts of its own row, which is fine: it isn&#39;t a violation of the convention above.</p><p>Stage Summary Table (<code>src/view/widgets/StageTable.tsx</code>) defaults to <strong>problem stages</strong>, with a header toggle to <strong>top 10 by duration</strong>. It renders inside the Full app report tab, not as a standalone board section.</p><h3 id="full-render-sequence" tabindex="-1">Full render sequence <a class="header-anchor" href="#full-render-sequence" aria-label="Permalink to &quot;Full render sequence&quot;">​</a></h3><p>Top to bottom, in <code>Dashboard.tsx</code>&#39;s <code>FilteredBoard</code>:</p><ol><li><code>FindingFilterBar</code> (plus <code>NoMatchBanner</code> when the active filter empties both finding streams).</li><li><code>Scorecard</code>: a three-tile run-info row (Wall-clock, Efficiency, Wastage), rendered once regardless of which tab is active.</li><li>A two-tab <code>Tabs</code> (skipped entirely when the active filter empties both finding streams; <code>NoMatchBanner</code> above already covers that case), tab labels <strong>Findings</strong> and <strong>Full app report</strong>: <ul><li><strong>Findings</strong> (<code>ImpactBoard</code>): a <code>HighestImpactBar</code> callout for the single highest-impact eligible finding, if any; then one <code>&lt;section&gt;</code> per impact band in <code>Critical</code> → <code>Warning</code> → <code>Info</code> order, each rendering nothing when it has neither a recommendation row nor an active widget: a recommendation-rollup <code>Table</code> (one row per eligible-finding type, <code>REGISTRY</code>-mapped, <code>incompleteRun</code> excluded: a direct row for a type with one finding, an expandable, paginated summary row for a type with more than one) followed by a <code>WidgetGrid</code> of that band&#39;s active <code>REGISTRY</code> widgets (ranked by widget order: ascending <code>DETECTORS</code> order, action-region components ahead of reference-region ones); then, below every impact band, a small always-visible grid of just Core Usage by Locality, mounted unconditionally regardless of finding state, and a collapsed &quot;Clean checks&quot; disclosure of <code>CleanCheckRow</code> lines built per detector type (every remaining <code>REGISTRY</code> key with zero findings); or inline &quot;No findings to fix right now.&quot; text when nothing is eligible at all.</li><li><strong>Full app report</strong> (<code>ReferenceSection</code>): WallClock → Timeline → Executor Count Over Time → StageTable → Evidence availability → fixed report-lens tail (ETL Phase Attribution → What-If Executor Scaling → Compute Efficiency → Wasted Core-Hours → Core-Usage Distribution). Structural-only, as above. Fully unmounted while Findings is active (see &quot;Render order&quot; above).</li></ul></li></ol><p>Report lenses carry no impact-band chip and self-hide when their input data is absent. Every widget self-wraps in <code>WidgetCard</code> (<code>src/view/WidgetCard.tsx</code>: an <code>&lt;h3&gt;</code> title, one level below the board&#39;s <code>&lt;h2&gt;</code> section headers) for the document outline, except Scorecard, which renders its own header band.</p><p>Every card is collapsible: the title sits in a <code>CollapsibleTrigger</code> button with a chevron (<code>WidgetCard.tsx</code>), and each widget sets its own <code>defaultCollapsed</code> (most default to <code>true</code>; <code>ImpactBoard</code>&#39;s action-region grid forces <code>defaultCollapsed</code> on every <code>REGISTRY</code> widget instance it mounts, so a marginal finding doesn&#39;t default open). Route navigation (&quot;jump to this finding&quot;) goes through the grid coordinator instead of a fixed <code>tabIndex</code>: <code>WidgetGrid.tsx</code> tracks each card&#39;s disclosure button and bumps <code>openRequestGeneration</code> to force a collapsed card open and focus its trigger when the target finding has no anchored row of its own. How much a card shows beyond that is a board-wide choice, set by the Basic/Advanced density tier in the topbar.</p>',43)])])}const u=o(i,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.js

@@ -1,6 +0,0 @@
-import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),c={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,i,n,l,p){return t(),d("div",null,[...e[0]||(e[0]=[a(`<h1 id="worker-protocol" tabindex="-1">Worker protocol <a class="header-anchor" href="#worker-protocol" aria-label="Permalink to &quot;Worker protocol&quot;">​</a></h1><h2 id="message-protocol" tabindex="-1">Message protocol <a class="header-anchor" href="#message-protocol" aria-label="Permalink to &quot;Message protocol&quot;">​</a></h2><p>Worker to main:</p><ul><li><code>app</code>, <code>stage</code>, <code>sql</code>, <code>executor</code>, <code>job</code>, <code>progress</code>: progressive, posted as the parser reads.</li><li><code>sqlPlan</code> (<code>{ type: &#39;sqlPlan&#39;, data: { executionId, planTree } }</code>): posted by <code>endSqlExecution</code> once a SQL execution&#39;s plan tree resolves. A repeated <code>SQLExecutionEnd</code> for that execution posts nothing, so no later <code>sql</code> message replaces the entry holding its <code>planTree</code>.</li><li><code>runAggregates</code>: one whole-run core-time-series summary (busy-core-ms, peak concurrency, per-stage task-duration sums), emitted just before <code>done</code>.</li><li><code>stageExecutorMetrics</code>: the post-completion re-post described in <a href="./overview.html#streaming">Streaming</a>, also emitted just before <code>done</code>.</li><li><code>done</code>, <code>taskData</code>, <code>error</code>.</li></ul><p>A History Server failure is always the typed, display-safe payload <code>{ type: &#39;error&#39;, source: &#39;shs&#39;, code }</code>, where <code>code</code> is one of <code>local-server-unavailable</code>, <code>upstream-unreachable</code>, <code>application-not-found</code>, <code>access-or-upstream-failure</code>, or <code>invalid-event-log</code>. It never carries an upstream message, URL, status, or response body.</p><p>Evidence availability has no dedicated worker message: the final <code>app</code> message carries a compact <code>evidenceInputs</code> counter summary, and <code>done.skippedLines</code> supplies its parse-integrity input.</p><p>Main to worker: <code>parse</code>, <code>parseFiles(files)</code> (rolling <code>eventlog_v2_*</code> directories, one continuous stream across files), <code>parseFromUrl({ baseUrl, appId, attemptId })</code>, and <code>getTaskData(stageId)</code>. The SHS request object is normalized before it reaches the worker: <code>{ baseUrl: string, appId: string, attemptId: string | null }</code>.</p><p><code>taskData</code> uses structured-clone (<code>.slice()</code>) so the worker retains its own <code>Float64Array</code> for re-renders.</p><p>Post-parse prefetch: after <code>done</code>, main runs <code>analyzer.ts</code> to build the bottleneck catalog, then fires parallel <code>getTaskData</code> for every flagged stage so their widgets render immediately. Unflagged stages are on-demand.</p><h3 id="decompress-worker" tabindex="-1">Decompress worker <a class="header-anchor" href="#decompress-worker" aria-label="Permalink to &quot;Decompress worker&quot;">​</a></h3><p>A dropped zstd file (<code>parse</code>, and each zstd file of a <code>parseFiles</code> directory) is decompressed in a second, nested worker, <code>packages/core/src/zstd-worker.ts</code>, so fzstd and the NDJSON parser run at the same time. On the largest real log, fzstd had been about 47% of the parse worker&#39;s time. The parse worker starts it on the first zstd file and reuses it for the rest of the parse. It dies with the parse worker, so the page&#39;s <code>terminate()</code> also cancels it. Other codecs and the SHS path (<code>parseFromUrl</code>) still decompress on the parse worker. A dropped History Server zip (<code>parse</code>) streams its zstd entries through the decompress worker too.</p><p><code>packages/core/src/zstd-worker-client.ts</code> is the parse-worker end: it plugs into <code>streamFile</code> as the <code>zstdDecoder</code> option, like the Node CLI&#39;s native decoder. The messages, all buffers moved by transfer, never copied:</p><ul><li>Parse to decompress: <code>start { id }</code> opens a stream, and replaces any open one. <code>data { id, seq, bytes, final }</code> carries one compressed read slice. <code>cancel { id }</code> drops the stream.</li><li>Decompress to parse: <code>ready</code> once, at startup. <code>chunk { id, bytes, length }</code> carries decoded output in batches of up to 1 MiB. <code>consumed { id, seq }</code> follows the last <code>chunk</code> of slice <code>seq</code>. <code>error { id, message }</code> ends the stream, and its message becomes the usual <code>Could not decompress</code> error.</li></ul><p>Flow control is a window of three input slices: <code>push()</code> resolves while fewer than three slices wait for their <code>consumed</code>. Each <code>chunk</code> is parsed in its message handler, so a slice is acknowledged only after its output was parsed, and at most three slices&#39; output is ever queued. The final <code>push()</code> resolves once every slice is acknowledged. When <code>streamFile</code> gives up mid-stream (a failed file read, a parse exception), it calls <code>cancel()</code>.</p><p>When the nested worker cannot start (<code>new Worker</code> throws or its script fails to load), the parse worker logs a warning and decodes with in-thread fzstd, as it did before, with identical output. A crash after startup fails the stream it was decoding, and later streams fall back the same way. The progress <code>pct</code> is the read position, so it can run up to the window ahead of the slice being parsed. The self-contained <code>file://</code> export never parses in the browser (it opens with its run already analyzed), so it never starts either worker.</p><h3 id="evidence-availability-worker-input" tabindex="-1">Evidence-availability worker input <a class="header-anchor" href="#evidence-availability-worker-input" aria-label="Permalink to &quot;Evidence-availability worker input&quot;">​</a></h3><p><code>packages/core/src/event-handlers.ts</code>&#39;s <code>createState()</code>/per-event handlers start and increment this fixed <code>evidenceInputs</code> shape on the normalized app model:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>environmentUpdates, applicationEnds, stageSubmissions, rddStorageSnapshots,</span></span>
-<span class="line"><span>sqlExecutions, resolvedSqlPlans, executorMetricRows, taskRecords</span></span></code></pre></div><p>The counters are a structured-clone-safe summary: they contain counts, not raw events, task records, host names, SQL text, paths, or Spark-property values. The final <code>app</code> message snapshots the counters after all input is processed, immediately before <code>done</code>; a consumer must not add a second worker message just for the ledger. <code>app</code> messages are also emitted mid-parse from <code>SparkListenerApplicationStart</code> and <code>SparkListenerEnvironmentUpdate</code>; those snapshots are partial and non-authoritative. Only the terminal <code>app</code> message emitted by <code>emitParseCompletion</code> (the one immediately before <code>done</code>) should be used for evidence-availability conclusions. On the main thread, <code>useIngest</code> combines the normalized <code>AppModel</code> with <code>done.skippedLines</code>, derives the ledger before calling <code>analyze()</code>, and stores it as <code>appModel.evidenceAvailability</code>.</p><h3 id="evidence-availability-contract-v1" tabindex="-1">Evidence-availability contract (V1) <a class="header-anchor" href="#evidence-availability-contract-v1" aria-label="Permalink to &quot;Evidence-availability contract (V1)&quot;">​</a></h3><p><code>packages/core/src/evidence-availability.ts</code> is the single reusable taxonomy for the browser, future report output, and future headless consumers. Its serialized ledger shape is <code>{ schemaVersion: 1, entries: EvidenceAvailabilityEntry[] }</code>. Entries are ordered by the following fixed eight-key enum:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>executorMetrics, rddStorageSnapshots, sqlPlan, sparkConfiguration,</span></span>
-<span class="line"><span>taskCoreTime, infrastructureContext, sourceContext, costContext</span></span></code></pre></div><p>Each entry has the stable fields <code>key</code>, <code>state</code>, <code>reasonCode</code>, <code>summary</code>, and optional <code>evidence</code>. The closed V1 state enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>present, disabled, notEmitted, notApplicable, outsideEventLog, unknown</span></span></code></pre></div><p>The closed V1 reason-code enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>observed, explicitlyDisabled, noObservedExecutorMetrics,</span></span>
-<span class="line"><span>noObservedStageSubmission, noRddStorageSnapshot, noResolvedSqlPlan,</span></span>
-<span class="line"><span>noSqlExecution, noEnvironmentUpdate, noTaskRecords, noUsableCoreTimeAggregate,</span></span>
-<span class="line"><span>outsideEventLogScope, parseIncomplete</span></span></code></pre></div><p><code>summary</code> is domain-agnostic presentation copy derived from the reason code; the enum values, not labels, are the machine contract. Optional <code>evidence</code> is limited to safe compact provenance of the exact <code>{ eventType, count }</code> shape. <code>eventType</code> values are pinned to the exact <code>evidenceInputs</code> counter keys; they are not shortened or aliased. It must never include raw Spark-property values, host names, paths, SQL text, task records, raw-event identifiers, or other event payloads.</p><table tabindex="0"><thead><tr><th>Key</th><th>Proven <code>present</code> evidence</th><th>Trustworthy absence / boundary result</th></tr></thead><tbody><tr><td><code>executorMetrics</code></td><td><code>executorMetricRows &gt; 0</code></td><td>Explicit observed <code>spark.eventLog.logStageExecutorMetrics=false</code> is <code>disabled</code> / <code>explicitlyDisabled</code>; otherwise <code>notEmitted</code> / <code>noObservedExecutorMetrics</code>. Observed metrics win over the explicit setting.</td></tr><tr><td><code>rddStorageSnapshots</code></td><td><code>rddStorageSnapshots &gt; 0</code> from <code>SparkListenerStageSubmitted</code></td><td>No stage submissions is <code>notEmitted</code> / <code>noObservedStageSubmission</code>; stages submitted without any <code>RDD Info</code> rows is <code>notEmitted</code> / <code>noRddStorageSnapshot</code>. This is RDD snapshot evidence, not block-update telemetry.</td></tr><tr><td><code>sqlPlan</code></td><td><code>resolvedSqlPlans &gt; 0</code></td><td>A SQL execution without a resolved plan is <code>notEmitted</code> / <code>noResolvedSqlPlan</code>; no SQL execution is <code>notApplicable</code> / <code>noSqlExecution</code>.</td></tr><tr><td><code>sparkConfiguration</code></td><td><code>environmentUpdates &gt; 0</code></td><td><code>notEmitted</code> / <code>noEnvironmentUpdate</code>; a normalized empty config object alone is not evidence.</td></tr><tr><td><code>taskCoreTime</code></td><td><code>taskRecords &gt; 0</code> and a usable <code>runAggregates.perStage</code> task count</td><td>No task records is <code>notEmitted</code> / <code>noTaskRecords</code>; task records with no usable aggregate are <code>notEmitted</code> / <code>noUsableCoreTimeAggregate</code>. An empty aggregate object is not usable evidence.</td></tr><tr><td><code>infrastructureContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>sourceContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>costContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr></tbody></table><p>Parse integrity is fail-closed. A parse is trustworthy only when it skipped no malformed lines and observed an application-end event. For an untrustworthy parse, proof already observed remains <code>present</code>, and an explicitly observed executor-metrics disablement remains <code>disabled</code>; every conclusion based on absence or irrelevance becomes <code>unknown</code> / <code>parseIncomplete</code>. The three outside-event-log product boundaries remain <code>outsideEventLog</code>. In particular, <code>notEmitted</code> and <code>notApplicable</code> are never favorable measurements inferred from an incomplete parse.</p><p>Availability is model metadata, not a detector or finding: it never changes detector thresholds or ordering, impact band, catalog/scorecard counts, no-bottleneck behavior, or detector suppression. <code>session-snapshot.ts</code> captures and restores <code>evidenceAvailability</code> with the normalized model, so a recent-file switch retains the same ledger without reparsing.</p><h3 id="portable-evidence-report-v1" tabindex="-1">Portable evidence report (V1) <a class="header-anchor" href="#portable-evidence-report-v1" aria-label="Permalink to &quot;Portable evidence report (V1)&quot;">​</a></h3><p><code>packages/core/src/evidence-report.ts</code>&#39;s <code>buildEvidenceReport(appModel, { redact })</code> returns <code>{ markdown, json }</code>: a self-contained, byte-stable document that runs the detectors over an <code>appModel</code> and serializes the result for sharing outside the tool. Raw task records are never included; identifier redaction (app id + host names → <code>app-1</code>/<code>host-1</code> pseudonyms via <code>packages/core/src/redact.ts</code>) is opt-in with <code>{ redact: true }</code>. The JSON is pinned by <code>EVIDENCE_SCHEMA_VERSION</code> (currently <code>3</code>, surfaced as <code>json.schemaVersion</code>) and has this fixed top-level key order:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>schemaVersion, summary, evidenceAvailability, detectors, findings, recommendations, cleanChecks</span></span></code></pre></div><ul><li><code>summary</code> is the run header: <code>{ app: { id, name, sparkVersion }, stageCount, jobCount, sqlExecutionCount, findingCount, impactBandCounts }</code>.</li><li><code>evidenceAvailability</code> is the ledger above (or <code>null</code> when absent).</li><li><code>detectors</code> is <code>detectorCatalog()</code> output: one <code>{ type, version, scope, thresholds, docAnchor }</code> per detector, in <code>DETECTORS</code> order, so the exact threshold set that produced each finding travels with the evidence.</li><li><code>findings</code> are deterministically sorted rows (impact band → type → stage → id), each with a stable <code>id</code>, <code>tag</code>, core columns, an always-present <code>actionLabel</code>, and an <code>evidence</code> sub-object for non-core fields; <code>confidence</code>/ <code>validationRequired</code>/<code>docAnchor</code> appear only when the detector emitted them.</li><li><code>recommendations</code> is the impact-ranked <code>buildRecommendationRollup</code> output (<code>packages/core/src/recommendation-rollup.ts</code>), and <code>cleanChecks</code> lists every detector type that fired zero findings this run: see the 2026-09-03 update below.</li></ul><p>Determinism holds because detector order, finding sort, and object key order are all fixed, so a given <code>appModel</code> serializes identically across calls.</p><p>Redaction enumerates hosts two ways: by walking the findings tree for every string value under a key literally named <code>host</code> (<code>evidence.host</code>, <code>evidence.failedTaskDetails[].host</code>, <code>evidence.retriedTaskDetails[].host</code>, and any future nested <code>host</code> field, all covered without enumerating paths), and by scanning every string value for EC2-style hostnames / bare IPv4 tokens. So identifiers that surface only in free text (recommendation copy, a <code>stageFailed</code> failure-reason value) are pseudonymized too. Pseudonym numbering uses a numeric-aware sort, so re-redacting an already-redacted report is a no-op even past <code>host-10</code>.</p><p>Failed-task error text is dropped rather than pseudonymized, since a message or stack trace can carry file paths and data values that no host pattern matches: every array under a key named <code>failureGroups</code> has its <code>message</code> replaced and the message text stripped from its <code>stackExcerpt</code> (<code>redactTaskFailureGroup</code> in <code>task-failure.ts</code>). That covers the evidence report and both the findings and the stage records of the HTML export.</p><p>The Markdown rendering mirrors the JSON&#39;s AC3 field set: each finding block prints its <code>detector version</code>, its sorted <code>evidence</code> entries (byte-magnitude keys humanized), and the report ends with a <code>## Detectors</code> catalog carrying the version + threshold set. A finding&#39;s <code>impactEstimate</code> (when its <code>basis</code> isn&#39;t <code>&#39;informational&#39;</code>) prints as its own <code>- impact: </code> line (<code>Estimated &lt;low&gt;-&lt;high&gt;</code> and/or the raw-waste figure, plus <code>estimateMethod</code>), via <code>renderImpactEstimate</code>/<code>formatWallClockRange</code>/<code>formatRawWaste</code> in <code>packages/core/src/evidence-report.ts</code>. <code>EvidenceExport</code> names downloads <code>evidence-&lt;appId&gt;[-redacted].&lt;md|json&gt;</code>, taking the app id from the (already pseudonymized when redacting) report so a redacted file never leaks the real id and is never name-identical to a raw export.</p><p>Decision 9 (design spec): <code>Finding</code>&#39;s <code>impactEstimate</code> field, plus the <code>utilizationFraction</code>/<code>memorySize</code>/<code>diskSize</code>/<code>numCachedPartitions</code>/<code>numPartitions</code> instrumentation fields the impact estimator reads, were added without bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>1</code>. Both additions are purely optional and ride <code>Finding</code>&#39;s existing optional-field-plus-catch-all convention, so an evidence report built before these fields existed still deserializes and compares byte-for-byte against one built after: nothing about the schema&#39;s stability guarantee changed, only its surface grew. A future reader who notices <code>impactEstimate</code> in the JSON without a schema bump is looking at this deliberate call, not an oversight.</p><p>2026-08-30 update: <code>EVIDENCE_SCHEMA_VERSION</code> was bumped to <code>2</code> for the occupancy-weighted attribution redesign (see <a href="./impact-estimation.html#occupancy-weighted-attribution">Occupancy-weighted attribution</a>): <code>ImpactEstimate</code>&#39;s shape changed from <code>{low, high}</code> to <code>{basis, wallClock, estimateMethod, rawWaste?}</code>, a real, non-additive breaking change to a field this same Decision 9 previously shipped without a bump. <code>impactEstimate</code> had zero consumers outside <code>packages/core/src/impact-estimator.ts</code> and its own tests at the time of this bump (confirmed by grep across <code>src/view/*</code> and <code>packages/core/src/evidence-report.ts</code>), so no other code needed migrating alongside it.</p><p><code>FindingRow</code>&#39;s schema was extended to surface <code>impactEstimate</code> as a first-class column without further bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, consistent with <code>Finding</code>&#39;s existing optional-field-plus-catch-all convention. It appears after the pinned core columns (<code>id</code>, <code>type</code>, <code>impactBand</code>, <code>stageId</code>, <code>metric</code>, <code>value</code>, <code>recommendation</code>, <code>detectorVersion</code>, plus optional but pinned <code>confidence</code>, <code>validationRequired</code>, <code>docAnchor</code>) without displacing any of them; byte-for-byte deserializability of existing reports is preserved. <code>FindingRow.impactEstimate</code> carries the full contract documented in <a href="./impact-estimation.html#occupancy-weighted-attribution">Impact estimation</a> (basis, wallClock, estimateMethod, rawWaste).</p><p>2026-09-03 update: the &quot;Fix These First&quot; dashboard redesign (impact-ranked recommendation rollup, short per-finding action labels, a clean-checks table) was UI-only when it shipped; this update ports the underlying data into <code>buildEvidenceReport()</code> so the CLI (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>) and the MCP tool (<code>diagnoseRun</code> in <code>packages/core/src/mcp-tools.ts</code>) get it too, not just the web markdown/JSON download. Three additions, all purely additive, so this does not bump <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, the same rationale as the <code>impactEstimate</code> addition immediately above: <code>FindingRow.actionLabel</code> is now always present (a short imperative label like &quot;Reduce shuffle size&quot;), sourced from a new core module, <code>packages/core/src/finding-action-label.ts</code>&#39;s <code>coreFindingActionLabel</code>, extracted from the (type, discriminant) switch statement that used to live only in the view layer (<code>src/view/finding-action-label.ts</code>, which now wraps the core function and layers its own <code>REGISTRY</code> fallback on top). <code>EvidenceReportJson.recommendations</code> is the same impact-ranked <code>buildRecommendationRollup</code> grouping (<code>packages/core/src/recommendation-rollup.ts</code>) that <code>FixTheseFirst.tsx</code> renders, so CLI/MCP/download consumers get the same &quot;what&#39;s the highest-impact fix&quot; ranking the dashboard shows, without changing the existing <code>findings</code> array&#39;s own impact-band-sorted order at all. <code>EvidenceReportJson.cleanChecks</code> lists every detector type that fired zero findings this run, each with <code>getThresholdSummary</code>&#39;s one-line &quot;what would have tripped it&quot; sentence; unlike the dashboard&#39;s <code>Alerts.tsx</code> clean-checks table, it deliberately includes the one &quot;always-mounted&quot; reference type (<code>coreLocality</code>) even when it has no findings, since a flat evidence report has no separate always-visible surface for it to already appear on the way that widget does on the board.</p><h3 id="finding-identity" tabindex="-1">Finding identity <a class="header-anchor" href="#finding-identity" aria-label="Permalink to &quot;Finding identity&quot;">​</a></h3><p>Every finding <code>analyzer.ts</code> emits carries a stable <code>id</code> (<code>push()</code>&#39;s <code>findingId()</code> choke point, FNV-1a hash over <code>type | location | metric | value | discriminators</code>) and a <code>detectorVersion</code> (from the emitting <code>DETECTORS</code> entry). The two are deliberately decoupled: <code>id</code> is derived only from a finding&#39;s evidence tuple, never from <code>detectorVersion</code>, so a threshold or logic tweak that bumps a detector&#39;s <code>version</code> does not change the <code>id</code> of findings it still emits at the same location/metric. A saved reference (dashboard bookmark, exported report row, future URL-restored filter) keeps pointing at the same logical finding across detector revisions; <code>detectorVersion</code> is separate provenance metadata for &quot;which ruleset produced this,&quot; not part of identity. See <code>packages/core/test/analyzer-finding-identity.test.js</code> for the decoupling proof.</p><p><code>id</code> stability holds for equivalent reruns of the same schema/analyzer version on the same input. Changing the id-derivation rule itself (the hash algorithm, or which fields feed the location key/discriminators) is an intentional breaking change and must bump <code>EVIDENCE_SCHEMA_VERSION</code> (<code>packages/core/src/evidence-report.ts</code>); there is no separate id-scheme version.</p><h3 id="headless-analysis-cli-v1" tabindex="-1">Headless analysis CLI (V1) <a class="header-anchor" href="#headless-analysis-cli-v1" aria-label="Permalink to &quot;Headless analysis CLI (V1)&quot;">​</a></h3><p><code>packages/cli/bin/sparkforensics-analyze.mjs</code> runs the same parser + detector contracts outside the browser, for CI. It accepts a single event-log file or a rolling-log directory, drives <code>runParse</code>/<code>runParseFiles</code> (<code>packages/core/src/parser-worker.ts</code>) with a Node-only File-like shim (<code>packages/core/src/cli/collect-run.ts</code>), and writes the exact <code>buildEvidenceReport</code> JSON schema described above: one format, not a second. Alternatively, <code>--shs-base-url &lt;url&gt; --app-id &lt;id&gt; [--attempt-id &lt;id&gt;]</code> fetches the run from a Spark History Server instead (mutually exclusive with the positional file/directory argument), calling <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, shared with the MCP server&#39;s SHS source path below) directly; no dependency on the <code>packages/server</code> package. A failed SHS fetch reports its message to <code>stderr</code> and exits <code>2</code>, same as a local file that can&#39;t be parsed.</p><p>Optional CLI-flag budgets (<code>--max-runtime &lt;ms&gt;</code>, <code>--max-spill &lt;gb&gt;</code>, <code>--max-skew &lt;ratio&gt;</code>, <code>--max-failed-task-rate &lt;pct&gt;</code>, <code>--min-efficiency &lt;pct&gt;</code>) are evaluated in <code>packages/core/src/cli/budgets.ts</code> against the existing finding catalog (<code>analyze()</code>) and <code>computeEfficiencyModel</code>; there is no second rule engine. A budget whose required evidence is missing (e.g. the run never emitted <code>ApplicationEnd</code>, or has no usable per-task <code>runAggregates</code>) is reported as inconclusive (<code>stderr</code> warning) rather than silently passing, and gets its own exit code distinct from both pass and violation. Exit codes: <code>0</code> pass, <code>1</code> a configured budget was violated, <code>2</code> bad arguments (unknown or value-less flag, unknown <code>--regression-metric</code> key) or input that could not be parsed at all, <code>3</code> no violations but at least one budget was inconclusive. A violation always wins over an inconclusive result in the same run (exit <code>1</code>, not <code>3</code>).</p><h3 id="mcp-server-v1" tabindex="-1">MCP server (V1) <a class="header-anchor" href="#mcp-server-v1" aria-label="Permalink to &quot;MCP server (V1)&quot;">​</a></h3><p><code>packages/core/src/mcp-server-factory.ts</code>&#39;s <code>createMcpServer()</code> registers 6 tools: <code>diagnose_run</code> (thresholded findings + remediation text), <code>get_run_summary</code> (app/stage/job/sql counts and duration, no findings), <code>compare_runs</code> (categorized findings delta + metric deltas between two runs), <code>evaluate_budgets</code> (pass/fail budget thresholds against one run, optionally with a second run for regression/fail-on-introduced budgets: the MCP side of the CLI&#39;s <code>evaluateBudgets()</code> gating), <code>get_finding_evidence</code> (raw evidence bundle for one finding, for drill-down after <code>diagnose_run</code>), and <code>get_finding_documentation</code> (detection/tuning reference docs for one finding type, independent of any run). None re-implement detector logic; all repackage <code>analyze</code>/<code>buildEvidenceReport</code>/<code>compareRuns</code>/<code>captureSnapshot</code>/<code>evaluateBudgets</code> from <code>packages/core/src/mcp-tools.ts</code>, which resolves a <code>source</code> (event-log <code>path</code>, or an SHS <code>shsBaseUrl</code>/<code>appId</code>/<code>attemptId</code> triple) into a cached <code>AppModel</code>. The SHS source path calls <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, also used directly by the CLI&#39;s <code>--shs-base-url</code> mode above), which reuses two extractions shared with the browser ingestion flow: <code>fetchShsEventLog</code> (<code>packages/core/src/proxy.js</code>) fetches the zip, and <code>decodeShsArchive</code> (<code>packages/core/src/shs-fetch.ts</code>, re-exported from <code>parser-worker.ts</code>&#39;s barrel) decodes it into a parsed <code>AppModel</code>, the same split <code>runParseFromUrl</code> itself calls into.</p><p>Two transports connect to that one factory: <code>packages/mcp/bin/sparkforensics-mcp.mjs</code> (stdio, for local MCP clients) and <code>packages/server/index.js</code>&#39;s <code>/mcp</code> route (streamable HTTP, for the local server). The run cache (<code>packages/core/src/mcp-tools.ts</code>) is a module-level LRU (cap 8 and 15-minute idle TTL by default, overridable via <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>, lazily swept on access) keyed by resolved source (path mtime, or SHS baseUrl+appId+attemptId), so a client mints a <code>runId</code> once via <code>resolveOrCreateRun</code> and reuses it across subsequent tool calls instead of re-parsing.</p><p>Every tool failure comes back as <code>{isError: true, content: [...], structuredContent: {code}}</code>, never an HTTP-status-shaped error; <code>code</code> is one of the 5 existing SHS codes (<code>SHS_ERROR_CODES</code>, <code>packages/core/src/shs-request.js</code>) plus <code>run-not-found</code>, <code>finding-not-found</code>, <code>invalid-event-log</code> (also covers a nonexistent <code>path</code> source), and <code>archive-too-large</code> (SHS archive over the <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code> byte cap, default 1 GiB, because the MCP path buffers the whole archive in memory, unlike the streaming <code>/shs-proxy</code> route).</p><p><code>scripts/vendor-core.mjs</code> (shared by <code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code>&#39;s <code>prepack</code> scripts) vendors <code>packages/core/src/</code> wholesale into each package&#39;s own <code>vendor-core/</code> at pack time, pre-stripping TypeScript to plain <code>.js</code> (Node&#39;s native TS stripping refuses to run on <code>.ts</code> files under <code>node_modules</code>, which is exactly where a published <code>vendor-core/</code> lands). Each package&#39;s bin/entry point resolves its needed module from <code>vendor-core/</code> if present, else falls back to the real <code>packages/core/src/</code> sibling loaded as <code>.ts</code> directly (<code>packages/server/index.js</code> mirrors <code>resolveStaticRoot</code>&#39;s <code>public/</code>-vs-<code>../dist</code> pattern for this same fallback).</p><h2 id="typescript-core-and-runtime-event-validation" tabindex="-1">TypeScript core and runtime event validation <a class="header-anchor" href="#typescript-core-and-runtime-event-validation" aria-label="Permalink to &quot;TypeScript core and runtime event validation&quot;">​</a></h2><p>All of <code>src/</code> (browser SPA) and <code>packages/core/src/</code> (shared analysis logic) is strict TypeScript; the only plain-<code>.js</code> holdouts are the two vendored third-party decompressors, <code>packages/core/src/vendor/fflate.js</code> and <code>packages/core/src/vendor/fzstd.js</code> (left untouched deliberately: vendored code, not project code). Every remaining import of a same-repo module uses a <code>.ts</code> specifier (e.g. <code>import { dispatchLine } from &#39;./event-handlers.ts&#39;</code>), not <code>.js</code>: the CLI and MCP entrypoints (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>, <code>packages/mcp/bin/sparkforensics-mcp.mjs</code>) run under plain Node&#39;s ESM resolver, which cannot remap a <code>.js</code> specifier to a same-named <code>.ts</code> file the way Vite/Vitest&#39;s bundler-style resolver can. <code>tsconfig.json</code> sets <code>&quot;allowImportingTsExtensions&quot;: true</code> to make this legal.</p><p>Event schema validation lives in <code>packages/core/src/event-schemas.ts</code>: one zod schema per <code>SparkListener*</code> event variant the parser understands (15 total: <code>LogStart</code>, <code>ApplicationStart</code>, <code>EnvironmentUpdate</code>, <code>ApplicationEnd</code>, <code>JobStart</code>, <code>JobEnd</code>, <code>StageSubmitted</code>, <code>StageCompleted</code>, <code>StageExecutorMetrics</code>, <code>TaskEnd</code>, the three SQL-execution-UI listener events, <code>ExecutorAdded</code>, <code>ExecutorRemoved</code>), combined into <code>SparkEventSchema = z.discriminatedUnion(&#39;Event&#39;, [...])</code>. <code>processEvent</code>&#39;s switch (<code>event-handlers.ts</code>) consumes the resulting <code>SparkEvent</code> union type directly, so a schema change and a handler&#39;s expectations can&#39;t silently drift apart.</p><p>The recursive <code>sparkPlanInfo.children</code> tree carried on SQL-execution-start events is validated by <code>parseSparkPlanInfoTree</code>, an iterative, explicit heap-allocated-stack parser, deliberately not <code>z.lazy()</code>. A real plan tree&#39;s depth is unbounded, and attacker-uncontrolled input should never hand zod&#39;s own recursive schema resolution an arbitrarily deep structure to walk. The iterative parser shallow-validates one node at a time via <code>z.object(...)</code> and builds the tree itself, capping at <code>MAX_PLAN_DEPTH = 500</code> (throws past that). This mirrors <code>packages/core/src/plan-tree-walk.ts</code>&#39;s <code>walkPlanTree</code>, which uses the same iterative-over-recursive approach for the <em>resolved</em> tree; <code>event-schemas.ts</code> applies it one layer earlier, to the raw JSON before it becomes a tree at all.</p><p>There are two external-data boundaries, the two places this codebase parses data it does not control. Both now run that data through a schema, and both treat a validation failure the same way: a silent skip, not a distinct error.</p><ul><li><code>dispatchLine</code> (<code>event-handlers.ts</code>): after <code>JSON.parse</code> succeeds, a line whose <code>Event</code> value is one of the 15 modeled types but fails that type&#39;s own schema now increments <code>skippedLines</code> (previously: no shape validation existed at all, and a malformed event silently corrupted downstream state with no signal anywhere). An <code>Event</code> value outside the 15 modeled types is still silently ignored without incrementing <code>skippedLines</code>, unchanged from before migration (see the note below on why the broader design was rejected). One exception: an AQE update that a later update for the same open execution supersedes is never parsed (<code>deferAdaptiveUpdate</code>, see <a href="./detector-contract.html">the detector contract</a>), so a malformed superseded update is not counted.</li><li><code>shs-fetch.ts</code>: on a non-OK response from the local SHS proxy, the JSON error envelope is validated against <code>ShsProxyErrorBodySchema</code> (<code>packages/core/src/shs-schemas.ts</code>, <code>{ code: string }</code>, <code>.passthrough()</code>). A body that isn&#39;t valid JSON, or is JSON but fails that schema, falls back to the generic <code>access-or-upstream-failure</code> code: the same silent-skip treatment as a malformed log line, collapsing what could have been a separate &quot;malformed SHS response&quot; error code into the existing generic one.</li></ul><p>Neither boundary distinguishes &quot;malformed JSON&quot; from &quot;wrong shape&quot; from &quot;unrecognized variant&quot; in what it reports outward: all three collapse into the same skip/fallback path. That was a deliberate scope decision. The correction below says why the <em>line</em> boundary stops there rather than flagging every unrecognized event type.</p><blockquote><p>Correction made during migration, not part of the original design: an earlier draft counted <em>any</em> <code>Event</code> value outside the 15 modeled types toward <code>skippedLines</code>, not just ones that fail their own schema. Running that design against real Spark event logs (which always contain plenty of ordinary event types this tool has never modeled: <code>TaskStart</code>, <code>BlockManagerAdded</code>, <code>ExecutorMetricsUpdate</code>, and others, always silently ignored pre-migration) pushed <code>skippedLines</code> from 0 to the tens of thousands on completely healthy logs. That would have tripped <code>evidence-availability.ts</code>&#39;s fail-closed <code>trustworthy</code> gate and shown a false &quot;malformed JSON&quot; warning on every real file. Caught in review against real fixtures, not synthetic ones; fixed to the narrower rule described above before merging. <code>evidence-availability.ts</code>&#39;s gate itself (<code>trustworthy = skippedLines === 0 &amp;&amp; applicationEnds &gt; 0</code>) needed no code change once this was fixed: it was already correct, the input feeding it was not.</p></blockquote><p>The exhaustiveness convention is <code>packages/core/src/assert-never.ts</code>. <code>assertNever(x: never): never</code> throws at runtime and, more importantly, fails <code>tsc</code> at compile time if <code>x</code> is not actually <code>never</code>, i.e. if some case of a union type isn&#39;t handled. Used today at the <code>default</code> arm of <code>event-handlers.ts</code>&#39;s <code>processEvent</code> switch (over <code>SparkEvent</code>) and <code>analyzer.ts</code>&#39;s scope-dispatch switch (over a detector&#39;s <code>scope: &#39;stage&#39; | &#39;sql&#39; | &#39;app&#39; | &#39;config&#39;</code>). It is the project&#39;s standard pattern for any future exhaustive switch or dispatch over a closed union: add <code>default: return assertNever(x);</code> (or the closest non-returning equivalent) so that adding a new union member without updating every consumer becomes a compile error instead of a silent runtime gap.</p>`,62)])])}const m=o(c,[["render",s]]);export{u as __pageData,m as default};

package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.lean.js

@@ -1 +0,0 @@
-import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),c={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,i,n,l,p){return t(),d("div",null,[...e[0]||(e[0]=[a("",62)])])}const m=o(c,[["render",s]]);export{u as __pageData,m as default};

package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.js

@@ -1 +0,0 @@
-import{_ as s,o,c as a,a5 as i}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Contributing","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/contributing.md","filePath":"contributor-guide/contributing.md"}'),t={name:"contributor-guide/contributing.md"};function n(c,e,r,d,h,l){return o(),a("div",null,[...e[0]||(e[0]=[i('<h1 id="contributing" tabindex="-1">Contributing <a class="header-anchor" href="#contributing" aria-label="Permalink to &quot;Contributing&quot;">​</a></h1><h2 id="where-does-new-content-go" tabindex="-1">Where does new content go? <a class="header-anchor" href="#where-does-new-content-go" aria-label="Permalink to &quot;Where does new content go?&quot;">​</a></h2><ul><li><code>docs/</code> is for internal engineering records: ADRs, audits, competitive research. It stays flat, unstructured markdown, read by contributors through GitHub&#39;s own file viewer.</li><li><code>docs-site/</code> (this site) is the published documentation for users and contributors. If you&#39;re writing something a user or a first-time contributor should read, it belongs here, not in <code>docs/</code>.</li></ul><h2 id="linting-before-you-commit" tabindex="-1">Linting before you commit <a class="header-anchor" href="#linting-before-you-commit" aria-label="Permalink to &quot;Linting before you commit&quot;">​</a></h2><p><code>npm install</code> runs the <code>prepare</code> script, which points git at <code>.githooks/</code> (<code>git config core.hooksPath .githooks</code>). After that, every commit runs <code>.githooks/pre-commit</code>: it lints any staged <code>.js</code>/<code>.mjs</code> files with eslint and blocks the commit if linting fails. <code>git commit --no-verify</code> skips it. To run the same linter by hand, use <code>npm run lint</code>.</p><h2 id="changesets-and-releases" tabindex="-1">Changesets and releases <a class="header-anchor" href="#changesets-and-releases" aria-label="Permalink to &quot;Changesets and releases&quot;">​</a></h2><p>A changeset is required for a PR that touches <code>packages/cli</code>, <code>packages/mcp</code>, <code>packages/server</code>, <code>packages/analyze</code>, <code>packages/sparkforensics</code>, or the root site app (<code>src/</code>, <code>index.html</code>, <code>vite.config.ts</code>, or the root <code>package.json</code>). Run <code>npx changeset add</code>, answer its prompts, and commit the generated <code>.changeset/*.md</code> file. CI&#39;s <code>changeset</code> job (<code>scripts/check-changeset.sh</code>) fails the PR otherwise, with instructions to fix it. <code>docs-site/</code> and <code>docs/</code> are exempt: doc-only changes don&#39;t need one.</p><p>Needing a changeset doesn&#39;t mean getting published, though. Only <code>packages/cli</code>, <code>packages/mcp</code>, <code>packages/server</code>, and the two alias packages publish to npm. The aliases, <code>packages/analyze</code> (<code>sparkforensics-analyze</code>) and <code>packages/sparkforensics</code> (<code>sparkforensics</code>), hold no code of their own: each depends on <code>sparkforensics-cli</code> and runs its <code>sparkforensics-analyze</code> bin, so <code>npx sparkforensics-analyze</code> works outside a checkout and nobody else can publish under either name. <code>.changeset/config.json</code>&#39;s <code>fixed</code> group keeps their versions equal to <code>sparkforensics-cli</code>&#39;s, so every CLI release republishes them. The root site (<code>sparkforensics-web</code>) is <code>&quot;private&quot;: true</code> and never published; its changeset only bumps its <code>package.json</code> version and writes a <code>CHANGELOG.md</code> entry, via <code>.changeset/config.json</code>&#39;s <code>privatePackages.version</code> setting. <code>@sparkforensics/core</code> is the other private package but stays out of this entirely (<code>.changeset/config.json</code>&#39;s <code>ignore</code> list): it&#39;s vendored into cli/mcp by filesystem copy at pack time, not read as a real dependency, so versioning it would have no consumer.</p><p>A release itself is two merges, not one: merging your feature PR into <code>main</code> runs <code>.github/workflows/release.yml</code>, which opens (or updates) a &quot;Version Packages&quot; PR collecting all pending changesets. Nothing publishes yet. Merging <em>that</em> PR is what actually bumps versions, writes changelogs, and runs <code>npm publish</code> for whichever publishable packages had pending changesets.</p><h3 id="publishing-a-new-package-name-maintainers" tabindex="-1">Publishing a new package name (maintainers) <a class="header-anchor" href="#publishing-a-new-package-name-maintainers" aria-label="Permalink to &quot;Publishing a new package name (maintainers)&quot;">​</a></h3><p>The release workflow publishes through npm Trusted Publishing, which can&#39;t create a package name that doesn&#39;t exist on npmjs.com yet. A new publishable package needs one manual publish from a maintainer&#39;s npm account first, then a Trusted Publisher entry. All five current packages (<code>sparkforensics-cli</code>, <code>sparkforensics-mcp</code>, <code>sparkforensics-server</code>, <code>sparkforensics-analyze</code> and <code>sparkforensics</code>) are already set up; this only applies to a package added later. Until it&#39;s done, the release run fails at that package&#39;s publish step (the other packages still publish).</p><ol><li><p>From a clean checkout of <code>main</code>, logged in with <code>npm login</code>, publish the new package at the version in its <code>package.json</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">cd</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> packages/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">new-packag</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">e</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> &amp;&amp; </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> publish</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --access</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> public</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span></code></pre></div></li><li><p>On npmjs.com, open the package&#39;s <strong>Settings</strong>, add a <strong>Trusted Publisher</strong> for GitHub Actions with organization <code>shuffle-works</code>, repository <code>sparkforensics</code>, workflow filename <code>release.yml</code> and no environment, and save.</p></li><li><p>Optional: in the same settings page, set publishing access to require two-factor authentication and disallow tokens, so only the workflow can publish from then on.</p></li></ol><p><code>scripts/publish-packages.mjs</code> skips a version that&#39;s already on the registry, so the next release run won&#39;t try to republish the bootstrap version.</p><p>For the full setup and test workflow, see <a href="./development-setup.html">Development setup</a> and <a href="./testing.html">Testing &amp; verification</a>.</p>',14)])])}const u=s(t,[["render",n]]);export{g as __pageData,u as default};

package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.js

@@ -1,15 +0,0 @@
-import{_ as i,o as e,c as a,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"Development setup","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/development-setup.md","filePath":"contributor-guide/development-setup.md"}'),n={name:"contributor-guide/development-setup.md"};function p(h,s,l,o,d,r){return e(),a("div",null,[...s[0]||(s[0]=[t(`<h1 id="development-setup" tabindex="-1">Development setup <a class="header-anchor" href="#development-setup" aria-label="Permalink to &quot;Development setup&quot;">​</a></h1><p>This page is for working on SparkForensics itself. To run the dashboard as a user, see <a href="./../user-guide/getting-started.html#local-server-mode">Run it locally</a>.</p><p>Clone the repo, then:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">git</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> submodule</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> update</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --init</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/log-corpus</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # or: git clone --recurse-submodules</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> install</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">              # vitest run: the root suite (site app and dev tooling)</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:watch</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">    # vitest, watch mode</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:core</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">     # packages/core&#39;s suite: detectors, analyzer, parser, MCP tools</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:cli</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">      # packages/cli&#39;s suite (packs and spawns the real tarball)</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:mcp</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">      # packages/mcp&#39;s suite (packs and spawns the real tarball)</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">           # Vite dev server for the app itself</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> build</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">         # production build, outputs dist/</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> preview</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">       # serves dist/ locally</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> tsc</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --noEmit</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">      # typecheck (strict TypeScript across src/ and packages/core/src/)</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> lint</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">          # eslint over the repo</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> packages/cli/bin/sparkforensics-analyze.mjs</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">rolling-log-dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt;   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># run the CLI analyzer against a log, outside the browser</span></span></code></pre></div><p><code>npm install</code> also runs the <code>prepare</code> script, which points git at <code>.githooks/</code>. After that, a pre-commit hook lints staged JS files on every commit. Fix what it reports, or skip it with <code>git commit --no-verify</code>.</p><p>The tuning reference under <code>packages/core/src/docs-content/{chapters,tuning,diagrams}</code> isn&#39;t committed. It&#39;s generated from the <code>shuffle-works/spark-tuning-reference</code> commit pinned in <code>packages/core/src/docs-content/upstream.json</code>: the first <code>npm test</code>, <code>npm run docs:dev</code> or <code>npm run build</code> fetches it over HTTPS (about 2s, no credentials) and later runs reuse the gitignored copy while it matches the pin. <code>npm run docs:fetch</code> does the same step on its own. Never edit those folders: fix the content upstream, then <code>npm run docs:bump</code> to move the pin. Running the MCP or CLI bins from source (<code>node packages/*/bin/...</code>) doesn&#39;t fetch it, so run <code>npm run docs:fetch</code> first: without it the MCP&#39;s tuning-reference lookups come back empty.</p><p>Offline, a cached copy keeps working. If the pin has moved since, <code>npm test</code> and <code>docs:dev</code> warn and use the older copy, while <code>docs:build</code>, <code>npm pack</code> and anything with <code>CI</code> set fail. To build from a local upstream checkout instead (offline, or to preview upstream edits in this docs site before they merge), set <code>SPARK_TUNING_REFERENCE_DIR</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">SPARK_TUNING_REFERENCE_DIR</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">../spark-tuning-reference</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:dev</span></span></code></pre></div><p>The same checks apply to that checkout. <code>npm pack</code> and CI accept it only when it sits at the pinned commit with no uncommitted <code>content/</code> changes.</p><p><code>dev/log-corpus</code> is a git submodule pointing at the public <code>spark-event-corpus-data</code> repo. It&#39;s optional locally: if you skip the <code>git submodule update</code> step above, the corpus-backed tests (such as <code>packages/server/test/shs-proxy-fixture.test.js</code>) report as skipped rather than failed, which is expected, not a bug. CI always checks it out and runs them, plus the <a href="./testing.html#corpus-regression-snapshot">corpus regression snapshot</a>.</p><p>The <code>packages/server/</code> package (the optional local-server deploy mode) keeps its own dependencies and test suite. Run it from the repo root: a second <code>npm install</code> inside <code>packages/server</code> prunes the shared root <code>node_modules</code>&#39;s workspace symlinks.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:server</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # runs packages/server&#39;s own suite, not part of \`npm test\`</span></span></code></pre></div><p>Packing or releasing <code>packages/server/</code> builds the docs site for you: its <code>prepack</code> runs <code>packages/server/scripts/copy-frontend.js</code>, which runs the root <code>npm run build</code> (<code>docs:build</code> included), so the packed artifact always ships <code>/docs/</code>.</p><p>This docs site has its own dev loop:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:dev</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">       # VitePress dev server for docs-site/</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:build</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">     # production build; also checks internal links/anchors</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:preview</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # serves the built docs site locally</span></span></code></pre></div>`,15)])])}const g=i(n,[["render",p]]);export{c as __pageData,g as default};

package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.lean.js

@@ -1 +0,0 @@
-import{_ as i,o as e,c as a,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"Development setup","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/development-setup.md","filePath":"contributor-guide/development-setup.md"}'),n={name:"contributor-guide/development-setup.md"};function p(h,s,l,o,d,r){return e(),a("div",null,[...s[0]||(s[0]=[t("",15)])])}const g=i(n,[["render",p]]);export{c as __pageData,g as default};

package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.js

@@ -1 +0,0 @@
-import{_ as t,o,c as s,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Testing & verification","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/testing.md","filePath":"contributor-guide/testing.md"}'),d={name:"contributor-guide/testing.md"};function i(c,e,n,r,l,p){return o(),s("div",null,[...e[0]||(e[0]=[a('<h1 id="testing-verification" tabindex="-1">Testing &amp; verification <a class="header-anchor" href="#testing-verification" aria-label="Permalink to &quot;Testing &amp; verification&quot;">​</a></h1><p>Before opening a PR:</p><ol><li>Run the full suite: <code>npm test</code> (root), plus <code>npm run test:core</code>, <code>npm run test:cli</code>, <code>npm run test:mcp</code> and <code>npm run test:server</code> for the packages you touched. Each of those four packages has its own Vitest config and suite, none of them covered by the root <code>npm test</code>.</li><li>Typecheck: <code>npx tsc --noEmit</code>.</li><li>Lint: <code>npm run lint</code>. This also runs automatically on every commit via <code>.githooks/pre-commit</code>, but run it yourself so you catch failures before committing.</li><li>For any UI change, verify it in a real browser (Playwright or headless Chrome). Unit and jsdom tests check code correctness; they don&#39;t prove a feature works end-to-end. Drive the app, exercise the golden path and the edge cases, and look for regressions elsewhere on the dashboard.</li><li>Report the passing test count when you&#39;re done, not &quot;tests pass.&quot;</li></ol><p>Sample Spark event logs for manual testing live outside this repo. Keep them in a sibling <code>../spark-log-examples/</code> directory: real <code>.zstd</code> event logs plus baseline/candidate pairs for run comparison.</p><h2 id="testing-layout" tabindex="-1">Testing layout <a class="header-anchor" href="#testing-layout" aria-label="Permalink to &quot;Testing layout&quot;">​</a></h2><p>Vitest throughout. Core logic tests (<code>packages/core/test/*.test.js</code>, covering parser, analyzer, detectors, format-utils, etc.) run under Node via <code>npm run test:core</code>. The root <code>npm test</code> runs two environments in one run: root-level tooling tests (<code>tests/*.test.js</code>) under Node, and view tests (<code>tests/view/*.test.tsx</code>), which declare <code>// @vitest-environment jsdom</code> and use React Testing Library (<code>render</code>/<code>screen</code>/<code>userEvent</code>). There are no hand-rolled DOM-mounting jsdom tests left: the old <code>tests/widgets/*.test.js</code> suite called <code>src/widgets/*.js</code>&#39;s <code>renderX(mount, ...)</code> functions directly, and every behavior it covered now has an RTL equivalent.</p><p><code>packages/cli</code>, <code>packages/mcp</code> and <code>packages/server</code> are separate npm packages, each with its own Vitest config and suite, not run by the root suite. Each has a pack-and-spawn test (for example <code>packages/server/test/bin.test.js</code>) that packs and installs the real published tarball and spawns the real <code>.bin</code> entry point, exercising the production build end-to-end.</p><p>Two contract tests guard invariants that a future edit could silently break. The &quot;detector contract&quot; suite in <code>packages/core/test/analyzer.test.js</code> asserts <code>stageSlowness</code> stays array-index-after <code>slowHost</code> in <code>DETECTORS</code> (see <a href="./architecture/detector-contract.html#detector-contract">Detector contract</a>). <code>tests/view/detector-registry.test.tsx</code> asserts <code>REGISTRY</code> completeness against every <code>DETECTORS</code> type, including the <code>underBroadcast</code>/<code>overBroadcast</code> correction: <code>broadcastSizing</code> itself is never an emitted <code>finding.type</code>, as the doc comment above <code>REGISTRY</code> in <code>src/view/detector-registry.tsx</code> explains.</p><h2 id="test-fixtures" tabindex="-1">Test fixtures <a class="header-anchor" href="#test-fixtures" aria-label="Permalink to &quot;Test fixtures&quot;">​</a></h2><p>Real Spark event logs from private workloads live under <code>examples/</code>, gitignored: they&#39;re large, and their file names and contents identify the jobs that produced them, so they never enter this public repo. Tests, docs and scripts refer to them only by the neutral labels below. A test that reads one skips when its file is missing, so the suite passes without any of them. To run those tests, copy or symlink your own logs into <code>examples/</code> under these names:</p><table tabindex="0"><thead><tr><th>Label</th><th>Log it stands for</th><th>Used by</th></tr></thead><tbody><tr><td><code>private-log-01.zstd</code></td><td>~12 MB zstd run with Plan Advisor findings</td><td><code>impact-estimator-real-log.test.js</code>, <code>plan-node-stage-mapping-real-log.test.js</code>, spot-checks in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-02.zstd</code></td><td>~2 MB zstd run with a heavy shuffle-and-spill stage</td><td><code>plan-node-stage-mapping-real-log.test.js</code>, spot-checks in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-03.zstd</code></td><td>~28 MB zstd run with about 1,200 stages</td><td>timing spot-check in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-04.zstd</code></td><td>~9 MB zstd run whose stage 394 plan became <code>plan-graph-stage-394.json</code></td><td><code>scripts/extract-plan-fixture.mjs</code></td></tr><tr><td><code>private-log-05</code></td><td>~76 MB uncompressed NDJSON run, 62 stages</td><td><code>parser-worker.test.js</code>; also a fast end-to-end smoke test</td></tr><tr><td><code>private-log-06</code></td><td>~1.3 GB uncompressed NDJSON run</td><td><code>detectors-plan.test.js</code>; also a stress test for the Web Worker streaming path</td></tr><tr><td><code>private-log-07</code></td><td>~120 MB log from a Spark History Server download</td><td>none: check the <code>LZ4Block</code> codec decoder by hand; <code>packages/core/test/lz4-block.test.js</code> uses a small hardcoded byte array instead</td></tr></tbody></table><p><code>scripts/extract-plan-fixture.mjs</code> anonymizes every table, column, path and literal before it writes the committed fixture; keep it that way if you extract another one.</p><p>Of the finding types moved onto the Gold Standard row/expand contract in the widget-gold-standard plan, only <code>retryWaste</code> and <code>autoscalingChurn</code> fire on any of the 15 real logs in <code>../spark-log-examples/</code>. <code>failures</code>, <code>stageFailed</code>, and <code>jobFailureRate</code> fire on none of them; they were verified with unit tests only (<code>tests/view/task-failures.test.tsx</code>, <code>tests/view/stage-failed.test.tsx</code>, <code>tests/view/job-failures.test.tsx</code>), not against a real log.</p><p><code>dev/log-corpus/</code> is a git submodule pointing at the public <code>spark-event-corpus-data</code> repo, pinned to a tag. It backs <code>packages/server/test/shs-proxy-fixture.test.js</code>, which loads a real <code>*-parquet-baseline.ndjson</code> fixture and asserts the <code>/shs-proxy</code> route streams it back unmodified. Populate it locally with <code>git submodule update --init dev/log-corpus</code>; without it, the test suite skips (see <a href="./development-setup.html">Development setup</a>). CI checks it out, so these tests run there.</p><h3 id="corpus-regression-snapshot" tabindex="-1">Corpus regression snapshot <a class="header-anchor" href="#corpus-regression-snapshot" aria-label="Permalink to &quot;Corpus regression snapshot&quot;">​</a></h3><p><code>dev/corpus-snapshot.json</code> records every finding (type, location, band, confidence, value, impact estimate) that the detectors produce on each public corpus log, plus its stage, SQL execution and skipped-line counts, with timings and memory left out so it is deterministic. CI&#39;s <code>core</code> job re-analyzes the corpus and fails on any difference, printing the added, removed, re-banded and re-estimated findings and changed counts per log:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/bench-analyze.mjs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --check</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/corpus-snapshot.json</span></span></code></pre></div><p>When a detector, threshold or estimator change is meant to alter findings, read that diff, confirm each change is the one you intended, then refresh the snapshot and commit it with the change:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/bench-analyze.mjs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --update</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/corpus-snapshot.json</span></span></code></pre></div><p>Both default to <code>dev/log-corpus/logs</code> and its <code>external/</code> folder. Only public corpus logs belong in the snapshot: never pass private logs to <code>--update</code>.</p><p>CI also runs a <code>node18</code> job, because the published <code>cli</code>, <code>mcp</code> and <code>server</code> packages declare <code>engines.node &gt;=18</code> while vitest needs Node 22+. It packs the three tarballs on <code>.nvmrc</code>&#39;s Node, installs them on Node 18, runs the CLI over every corpus log plus a zstd copy of one, checks that the MCP server answers <code>initialize</code>, and starts <code>sparkforensics-server</code> to check it serves the app and answers <code>initialize</code> on <code>/mcp</code>.</p>',21)])])}const g=t(d,[["render",i]]);export{u as __pageData,g as default};

package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as o,a5 as r}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Anti-Patterns","description":"","frontmatter":{"title":"Anti-Patterns"},"headers":[],"relativePath":"tuning-reference/anti-patterns.md","filePath":"tuning-reference/anti-patterns.md"}'),s={name:"tuning-reference/anti-patterns.md"};function n(i,e,f,l,c,d){return a(),o("div",null,[...e[0]||(e[0]=[r('<h1 id="anti-patterns" tabindex="-1">Anti-Patterns <a class="header-anchor" href="#anti-patterns" aria-label="Permalink to &quot;Anti-Patterns {#anti-patterns}&quot;">​</a></h1><p>Practitioner checklists on Spark performance converge on a recurring handful of mistakes rather than exotic ones. Each entry below covers what the mistake is, how to detect it, why it hurts, and how to fix it.</p><ol><li><p><strong>Staying on the RDD API instead of DataFrame/Dataset</strong></p><ul><li><strong>What:</strong> Building jobs on the RDD API (or RDD-based MLlib) instead of the DataFrame/Dataset API.</li><li><strong>Detect:</strong> Code that drives its logic through <code>.map</code>/<code>.filter</code> lambdas over RDDs, or that calls into the RDD-based MLlib API.</li><li><strong>Why:</strong> RDDs are lambda-driven, so Spark cannot see inside the closures. It can&#39;t apply predicate pushdown, filter reordering, Adaptive Query Execution, or cost-based join reordering. The RDD-based MLlib API is in maintenance mode for the same reason.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup></li><li><strong>Fix:</strong> Use the DataFrame/Dataset API so Catalyst can optimize the query plan.</li></ul></li><li><p><strong>Reading CSV/JSON without an explicit schema</strong></p><ul><li><strong>What:</strong> Reading raw CSV or JSON files without supplying an explicit schema.</li><li><strong>Detect:</strong> A read step that spends noticeable time before any real work starts, because Spark is inferring types from the data itself.</li><li><strong>Why:</strong> Schema inference forces a full scan of the data just to determine types, and CSV/JSON don&#39;t support the column pruning, predicate pushdown, or stats-based file skipping that a columnar format does.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></li><li><strong>Fix:</strong> Supply an explicit schema, or prefer Parquet for analytical workloads, where column pruning, predicate pushdown, and stats-based file skipping work out of the box.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup></li></ul></li><li><p><strong>Compressing input files with a non-splittable codec (GZIP)</strong></p><ul><li><strong>What:</strong> Storing input data as large GZIP files.</li><li><strong>Detect:</strong> One task/executor taking far longer than the rest on a stage reading GZIP-compressed input, while other executors sit idle.</li><li><strong>Why:</strong> A GZIP file can&#39;t be split across executors, so a single node has to decompress the whole file alone.<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup></li><li><strong>Fix:</strong> Use a splittable codec instead: Snappy, LZ4, or ZSTD.<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup></li></ul></li><li><p><strong>Storing data as JSON instead of a binary columnar/row format</strong></p><ul><li><strong>What:</strong> Persisting datasets as JSON rather than a binary columnar or row format.</li><li><strong>Detect:</strong> Read-heavy jobs where a large, recurring share of stage time goes to parsing the same JSON structures on every read.</li><li><strong>Why:</strong> JSON has to be re-parsed from text on every single read.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup></li><li><strong>Fix:</strong> Store data as Avro, Parquet, Thrift, or Protobuf structs in a sequence file to avoid the repeated parsing cost.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></li></ul></li><li><p><strong>Not registering custom classes with Kryo</strong></p><ul><li><strong>What:</strong> Using Kryo serialization without registering the application&#39;s custom classes.</li><li><strong>Detect:</strong> Larger-than-expected serialized record sizes, and poor efficiency when using a serialized cache storage level such as <code>MEMORY_SER</code>.</li><li><strong>Why:</strong> Unregistered classes increase serialized-record size and hurt serialized cache storage levels.<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup></li><li><strong>Fix:</strong> Register the application&#39;s custom classes with Kryo.</li></ul></li><li><p><strong>Calling <code>collect()</code> on a large DataFrame</strong></p><ul><li><p><strong>What:</strong> Materializing an entire DataFrame into driver memory with <code>collect()</code>.</p></li><li><p><strong>Detect:</strong> A driver <code>OutOfMemoryError</code>, or a job that aborts once <code>spark.driver.maxResultSize</code> (default 1 GB) is exceeded. Spark also logs a warning once a single task&#39;s serialized result passes roughly 1 MB.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup></p></li><li><p><strong>Why:</strong> <code>collect()</code> gathers every <code>Row</code> from every partition into a single in-memory <code>Array</code> that lives entirely on the driver, not the cluster. The driver JVM has to hold the whole result set in its own heap.<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup></p></li><li><p><strong>Fix:</strong> Use aggregations or <code>take(n)</code> instead of collecting the full result set.</p><blockquote><p><strong>PySpark:</strong> <code>toPandas()</code> performs the same driver-side collection as <code>collect()</code> for Python users, so avoid it on large DataFrames too.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup></p></blockquote></li></ul></li><li><p><strong>Assuming <code>cache()</code> + <code>count()</code> guarantees the DataFrame stays fully cached</strong></p><ul><li><strong>What:</strong> Treating <code>df.cache(); df.count()</code> as a guarantee that the DataFrame remains fully persisted for later actions.</li><li><strong>Detect:</strong> Later actions on a supposedly-cached DataFrame recompute from source instead of hitting the cache. There&#39;s no built-in visibility into how much of a DataFrame is still actually cached, so this typically only surfaces as unexpectedly slow recomputation.<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></li><li><strong>Why:</strong> <code>count()</code> does force every partition to be computed and attempted for caching, unlike <code>take</code>/<code>limit</code>, which only touch the partitions they need.<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> But partitions can&#39;t be fractionally cached: if there isn&#39;t room for all of them, the ones that don&#39;t fit are silently dropped and recomputed on next access.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup> Cached blocks also compete with execution for the same memory pool and can be evicted under pressure without warning, and caching is tied to the <em>analyzed</em> (pre-optimization) logical plan, so a semantically identical query with a different analyzed plan bypasses the cache entirely and recomputes from source.<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup> Losing an executor drops any cached block that wasn&#39;t stored with a replicated storage level like <code>MEMORY_AND_DISK_2</code>.<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup></li><li><strong>Fix:</strong> Keep using <code>.cache()</code> followed by <code>.count()</code> to force materialization, but don&#39;t assume that guarantees persistence (there&#39;s no built-in way to confirm how much of the DataFrame is actually cached<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>), and use a replicated storage level if losing a cached partition to executor failure would be costly.</li></ul></li><li><p><strong>Row-at-a-time Python UDFs</strong></p><ul><li><strong>What:</strong> Writing Python UDFs that operate one row at a time instead of vectorized Pandas UDFs.</li><li><strong>Detect:</strong> A UDF-heavy stage where most of the time goes to serialization rather than actual computation.<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup></li><li><strong>Why:</strong> The overhead is serialization plus per-row JVM↔Python data movement, not raw Python interpreter speed. Row-at-a-time UDFs &quot;suffer from high serialization and invocation overhead.&quot;<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> RDD-era Python UDFs pay a <em>double</em> serialization cost: Java/Scala objects are serialized, then re-serialized to Python via <code>cloudpickle</code> and back on every call.<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> A Databricks benchmark on a 10M-row DataFrame showed vectorized Pandas UDFs beating row-at-a-time UDFs &quot;across the board, ranging from 3x to over 100x.&quot;<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup></li><li><strong>Fix:</strong> Use vectorized Pandas UDFs instead of row-at-a-time UDFs: once data is transferred via Arrow, there&#39;s no need to serialize/pickle it, since it&#39;s already in a format consumable by the Python process.<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup></li></ul></li><li><p><strong>Leaving <code>spark.sql.shuffle.partitions</code> at its default of 200</strong></p><ul><li><strong>What:</strong> Never tuning <code>spark.sql.shuffle.partitions</code> away from its default of 200 for wide transformations (<code>join</code>, <code>groupBy</code>, aggregations).</li><li><strong>Detect:</strong> In the Spark UI Stages tab, too few partitions for the data size shows up as &quot;Spill (Memory)&quot; or &quot;Spill (Disk)&quot; entries against a stage; too many partitions for the data size shows up as a large task/stage count where each task&#39;s duration is dominated by scheduling and bookkeeping, often paired with a large number of tiny output files.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Why:</strong> 200 is a fixed default regardless of whether you&#39;re joining 5MB or 5TB.<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup><sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup> Too few partitions means each one holds more data than fits comfortably in executor memory, which increases load per executor and leads to spills once partition size exceeds available memory.<sup class="footnote-ref"><a href="#fn10" id="fnref10:2">[10:2]</a></sup> Too many partitions on a small dataset can shrink tasks to around ten rows each, so &quot;most of your CPUs will just be sitting there doing nothing.&quot;<sup class="footnote-ref"><a href="#fn10" id="fnref10:3">[10:3]</a></sup></li><li><strong>Fix:</strong> Increase <code>spark.sql.shuffle.partitions</code> if tasks are processing multiple GB each and spilling; decrease it if tasks finish in a couple of seconds and write tiny files. A rough starting target is 100–200 MB of data per task, tuned per dataset.<sup class="footnote-ref"><a href="#fn10" id="fnref10:4">[10:4]</a></sup></li></ul></li><li><p><strong>Broadcasting a table that&#39;s too large</strong></p><ul><li><strong>What:</strong> Forcing or allowing a broadcast join on a table that&#39;s too large to broadcast safely.</li><li><strong>Detect:</strong> A driver <code>OutOfMemoryError</code> during a broadcast join.</li><li><strong>Why:</strong> The small side of the join has to be collected onto the driver before it can be broadcast to every executor. That collection step is the same driver-memory operation as <code>.collect()</code>, and it&#39;s what fails, not executor-side replication or a serialization timeout: &quot;if you try to broadcast something too large, you can crash your driver node (because that collect is expensive).&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup></li><li><strong>Fix:</strong> Use <code>spark.sql.autoBroadcastJoinThreshold</code> to control the maximum size Spark will broadcast, or increase driver memory.<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup></li></ul></li><li><p><strong>Reaching for <code>coalesce(1)</code> before every single-file write</strong></p><ul><li><strong>What:</strong> Always using <code>coalesce(1)</code> instead of <code>repartition(1)</code> when a single output file is needed.</li><li><strong>Detect:</strong> The entire upstream stage (including filtering/transformation work that would otherwise run in parallel) collapses onto a single task/executor, leaving the rest of the cluster idle.</li><li><strong>Why:</strong> <code>coalesce</code> is a narrow transformation, so it &quot;causes the upstream partitions in the entire stage to execute with the level of parallelism assigned by coalesce,&quot; fusing all preceding work down to one task.<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup> <code>repartition</code> triggers a full shuffle instead, which inserts an explicit shuffle boundary so the upstream stage keeps its original parallelism.<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup></li><li><strong>Fix:</strong> Use <code>repartition(1)</code> when there is meaningful upstream computation you don&#39;t want collapsed onto a single executor; reserve <code>coalesce(1)</code> for when the upstream stage is already cheap and paying for a shuffle would be wasted cost.<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup></li></ul></li><li><p><strong>Not verifying predicate pushdown is actually happening</strong></p><ul><li><strong>What:</strong> Assuming filters and column projections are pushed down to the data source without checking the physical plan.</li><li><strong>Detect:</strong> Run <code>.explain()</code> and look at the <code>Scan</code> node for a <code>PushedFilters</code> marker. For example, <code>*Scan JDBCRel... PushedFilters: [*In(DEST_COUNTRY_NAME, [Anguilla, Sweden])]</code>. Column pruning is verified the same way: a <code>.select()</code> on one column should show a narrowed <code>ReadSchema</code> at the scan rather than a full-table scan.<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup></li><li><strong>Why:</strong> Spark pushes down simple filters (column equality, <code>IN</code>, <code>IS NULL</code>) to JDBC sources automatically, but anything with a computed column or cast won&#39;t push down and gets evaluated in Spark after the full read.<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup> Some sources also only partially handle a filter and leave Spark to re-evaluate it as a safety mechanism; that&#39;s still a &quot;good&quot; pushdown since the amount of data read is reduced.<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup></li><li><strong>Fix:</strong> Check <code>explain</code> output (paired with <code>printSchema</code>)<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup> for <code>PushedFilters</code> after writing a filter, and rewrite filters that use casts or computed columns as simple predicates where possible so they can push down.<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup></li></ul></li></ol><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Advanced Analytics with PySpark</em>, Tandon, Ryza, Laserson et al., ch. 2 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, chs. 5, 8, 9, 18, 19 <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, chs. 3, 5, 7 <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, chs. 5, 7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-dataframe-api" target="_blank" rel="noreferrer">Spark Tips: DataFrame API</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://www.databricks.com/blog/2017/10/30/introducing-vectorized-udfs-for-pyspark.html" target="_blank" rel="noreferrer">Introducing Vectorized UDFs for PySpark</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a> <a href="#fnref10:2" class="footnote-backref">↩︎</a> <a href="#fnref10:3" class="footnote-backref">↩︎</a> <a href="#fnref10:4" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li></ol></section>',6)])])}const u=t(s,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as o,a5 as r}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Anti-Patterns","description":"","frontmatter":{"title":"Anti-Patterns"},"headers":[],"relativePath":"tuning-reference/anti-patterns.md","filePath":"tuning-reference/anti-patterns.md"}'),s={name:"tuning-reference/anti-patterns.md"};function n(i,e,f,l,c,d){return a(),o("div",null,[...e[0]||(e[0]=[r("",6)])])}const u=t(s,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const n="/docs/assets/aqe-loop.IwQSATHw.svg",r="/docs/assets/aqe-loop.dark.DGbaxqJE.svg",m=JSON.parse('{"title":"Adaptive Query Execution","description":"","frontmatter":{"title":"Adaptive Query Execution"},"headers":[],"relativePath":"tuning-reference/aqe.md","filePath":"tuning-reference/aqe.md"}'),i={name:"tuning-reference/aqe.md"};function f(l,e,c,h,d,p){return t(),s("div",null,[...e[0]||(e[0]=[o('<h1 id="aqe" tabindex="-1">Adaptive Query Execution <a class="header-anchor" href="#aqe" aria-label="Permalink to &quot;Adaptive Query Execution {#aqe}&quot;">​</a></h1><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>The static Catalyst optimizer plans a query once, before execution, using cost estimates derived from static statistics: row counts, min/max, NDVs, or defaults when statistics are missing.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> Adaptive Query Execution (AQE), shipped in Spark 3.0, instead re-optimizes the plan mid-execution using runtime statistics gathered from completed &quot;query stages&quot;: the sections of a plan bounded by shuffle or broadcast exchange materialization points.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> A shuffle or broadcast forces Spark to materialize its input before continuing, which makes it a natural checkpoint: once one or more leaf stages finish, AQE marks them complete, updates the logical plan with the real (not estimated) statistics, and reruns a selected set of logical and physical optimization rules (including AQE-specific rules such as partition coalescing and skew-join handling) before executing the next stages.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></p><p>Shuffle statistics only become available once a stage is <em>fully</em> materialized, not incrementally as individual map tasks finish. A stage&#39;s successor can only proceed once every parallel process producing that stage&#39;s output has completed, which is exactly why materialization points are the reoptimization opportunity: it&#39;s the moment when statistics on all of a stage&#39;s partitions are known and the next stage hasn&#39;t started yet.<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup> AQE kicks off all leaf stages (the ones with no upstream dependency) first; as each one finishes, the framework marks it complete, updates the plan, and re-optimizes before launching whichever next stages now have all their children materialized. This execute→reoptimize→execute loop repeats (once per completed stage, not just once) until the whole query finishes, so the number of reoptimizations scales with how many shuffle/broadcast boundaries the plan has.<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup> The Spark SQL config description for <code>spark.sql.adaptive.enabled</code> frames this the same way: AQE re-optimizes &quot;the query plan in the middle of query execution, based on accurate runtime statistics.&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup></p><img class="light-only" src="'+n+'" alt="AQE runs a loop that executes leaf stages, materializes at a shuffle or broadcast boundary, collects runtime statistics, re-applies rules to coalesce partitions and split skew and promote joins to broadcast, then launches the next stages until the plan is complete."><img class="dark-only" src="'+r+'" alt="AQE runs a loop that executes leaf stages, materializes at a shuffle or broadcast boundary, collects runtime statistics, re-applies rules to coalesce partitions and split skew and promote joins to broadcast, then launches the next stages until the plan is complete."><p>AQE re-optimizes three things the static optimizer cannot, because none of them are knowable before execution: the number of post-shuffle partitions (coalescing small partitions produced by wide transformations), the join strategy (converting a statically planned sort-merge join to a broadcast join once the actual materialized size of a join side is known), and skewed partitions in a shuffle join (splitting oversized partitions detected from shuffle file statistics).<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup> High Performance Spark frames this as AQE using &quot;runtime information about the data it is processing along with the target output to go beyond static optimizations,&quot; with partitioning and join strategy singled out as the two biggest areas it impacts.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup> AQE also performs empty-relation propagation, replacing subqueries that turn out to be empty (impossible joins, empty unions) with a dummy empty <code>LocalRelation</code>, again something only knowable once data is materialized.<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Because these are runtime decisions, none of them show up in a static <code>explain()</code> call; you have to run the query and check the Spark UI for the finalized, adapted plan.<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup> The same caveat applies more broadly: since AQE adjusts plans at runtime, <code>explain()</code> might show one plan while the Spark UI shows a different one actually executed.<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup></p><p>What you&#39;re looking for in that adaptive plan are the specific runtime rewrites AQE can make:</p><ul><li><strong>Partition coalescing.</strong> AQE combines <em>adjacent</em> small post-shuffle partitions into bigger ones by reading shuffle file statistics,<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup> targeting <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code> (default 64 MB).<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> In a worked example, five post-shuffle partitions where three are small get coalesced into one, cutting the final-aggregation task count from five to three.<sup class="footnote-ref"><a href="#fn2" id="fnref2:6">[2:6]</a></sup></li><li><strong>Skew splitting.</strong> A partition is considered skewed if its size is larger than the median partition size times <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default 5.0) <em>and</em> larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default 256 MB).<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup><sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup> The original design doc&#39;s worked example (table A join table B, where B&#39;s partition 0 is skewed) has the <code>OptimizeSkewedJoin</code> rule divide that partition into N smaller splits reading disjoint ranges of upstream map outputs, create N separate tasks each joining its slice of B&#39;s partition 0 against A&#39;s corresponding partition 0, then union the N results. That trades reading A&#39;s partition 0 N times against eliminating a straggler task.<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup></li><li><strong>Join strategy promotion.</strong> When a join side&#39;s runtime-materialized size falls under the adaptive broadcast threshold, AQE converts a planned sort-merge join to a broadcast hash join. It reuses the shuffle output already written rather than re-materializing the build side. The perf-tuning guide frames the benefit as avoiding a re-sort of both join sides and reading the existing shuffle files locally instead, conditional on <code>spark.sql.adaptive.localShuffleReader.enabled</code>.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup> The same guide calls this conversion &quot;not as efficient as planning a broadcast hash join in the first place,&quot; consistent with reusing existing shuffle output rather than recomputing an equivalent build side from scratch.<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup></li><li><strong>Local shuffle reads.</strong> Once that promotion happens, the side that no longer needs to be partitioned by join key can be read straight off the shuffle files each executor already has locally, instead of pulling blocks from remote executors over the network. This is what <code>spark.sql.adaptive.localShuffleReader.enabled</code> (default true since 3.0.0) does, and it applies whenever shuffle partitioning is no longer needed, such as after a sort-merge-to- broadcast conversion.<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:7">[2:7]</a></sup></li></ul><p>Dynamic Partition Pruning is a separate mechanism worth distinguishing from AQE&#39;s loop when reading a plan: DPP inserts a <code>DynamicPruningSubquery</code>/<code>DynamicPruningExpression</code> node during logical optimization/physical planning, before execution starts, when an equi-join is on a partition column and pruning looks beneficial by static statistics.<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> Because DPP&#39;s calculation runs at planning time and AQE&#39;s runs later, mid-execution, off completed-stage statistics, the ordering implied is that DPP&#39;s pruning predicate is planned first, with AQE&#39;s adaptive rules applying afterward, during execution, on top of whatever DPP already pruned<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:8">[2:8]</a></sup>, though the exact current-version integration between the two isn&#39;t detailed further in the available sources, so treat that ordering as directionally supported rather than exhaustively confirmed.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>AQE exists precisely because partition sizing, join strategy, and skew are only knowable once real data has been materialized. That&#39;s the gap the static optimizer can&#39;t close on its own.<sup class="footnote-ref"><a href="#fn2" id="fnref2:9">[2:9]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup> But it isn&#39;t a guaranteed win. High Performance Spark warns that &quot;while AQE generally does better than unoptimized code, there are cases, especially when targeting Iceberg tables, where AQE does more harm than good&quot;: AQE&#39;s partitioning-to-target-table matching is generally beneficial but can cause severe regressions if you&#39;ve already done deliberate work to avoid key skew via custom partitioning, since AQE&#39;s attempt to match the target table&#39;s write partitioning can undo that work, especially when there&#39;s a large amount of key skew.<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup></p><p>A second, more general failure mode is bad input statistics feeding AQE&#39;s runtime decisions: if the source data has wrong stats (compressed JSON or Kafka streams are called out specifically), AQE can make things worse rather than better.<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup> The same source&#39;s framing is that AQE &quot;isn&#39;t magic... it helps polish your plan; it doesn&#39;t design it for you. You still need good partitioning fundamentals&quot;. AQE can smooth over a reasonably partitioned plan but doesn&#39;t reliably rescue one built on bad upstream statistics or layout.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup></p><p>AQE&#39;s runtime re-optimization also doesn&#39;t reach into DataFrame caching, which is worth knowing so you don&#39;t reach for the wrong lever: <code>.cache()</code>/<code>.persist()</code> wrap the query&#39;s <em>analyzed</em> logical plan (the point after the analyzer phase but before optimization) in an <code>InMemoryRelation</code> node, and cache lookup compares the analyzed plan of the current query against what was cached, recomputing if they don&#39;t match exactly, even if the two queries would ultimately produce the same optimized physical plan.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup> Because AQE operates on the physical plan at runtime, well downstream of that analyzed-plan cache key, disabling AQE doesn&#39;t change whether a cached DataFrame&#39;s analyzed plan matches; it only changes how the physical/execution plan is chosen after that cache lookup already happened.<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup></p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>AQE&#39;s coalescing, skew-handling, and join-promotion rules only fire once <code>spark.sql.adaptive.enabled</code> is on, and each has its own knobs worth tuning rather than leaving at the defaults:</p><ul><li><code>spark.sql.adaptive.coalescePartitions.enabled</code> (default true) turns on partition coalescing; <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code> (default 64 MB) sets its target size.<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup> <code>spark.sql.adaptive.coalescePartitions.minPartitionNum</code> bounds the minimum resulting parallelism,<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup> and <code>spark.sql.adaptive.coalescePartitions.minPartitionSize</code> (default 1 MB, since 3.2) sets a floor on coalesced partition size for when the adaptively calculated target is too small.<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup></li><li><code>spark.sql.adaptive.coalescePartitions.parallelismFirst</code> (default true, since 3.2) tells Spark to ignore the advisory target size entirely and instead calculate a (usually smaller) target based on cluster default parallelism, prioritizing task parallelism over hitting the 64 MB target. On a busy cluster, set it to <code>false</code> to respect the configured target size and avoid producing many small tasks.<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup></li><li><code>spark.sql.adaptive.skewJoin.enabled</code> (default true) turns on skew-join handling for both sort-merge and shuffled hash joins.<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> Tune the detection thresholds via <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default 5.0) and <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default 256 MB); set the byte threshold larger than <code>advisoryPartitionSizeInBytes</code>.<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup> Use <code>spark.sql.adaptive.forceOptimizeSkewedJoin</code> (default false, since 3.3.0) to force the rule to fire even when it would introduce extra shuffle.<sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup></li><li><code>spark.sql.adaptive.localShuffleReader.enabled</code> (default true, since 3.0.0) keeps the local, per-mapper shuffle read enabled after a sort-merge-to-broadcast conversion.<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup></li></ul><p>If AQE&#39;s target-table partition matching is regressing an Iceberg write that already has deliberate custom partitioning to avoid key skew, set the table&#39;s (or write-time) <code>write.distribution-mode</code> property to <code>none</code>, at the cost of a higher risk of many small files from unsorted/unhashed writers per partition.<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup></p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/04/13/deep-dive-into-spark-sqls-catalyst-optimizer.html" target="_blank" rel="noreferrer">Deep Dive into Spark SQL&#39;s Catalyst Optimizer</a> <a href="#fnref1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://www.databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html" target="_blank" rel="noreferrer">Adaptive Query Execution: Speeding Up Spark SQL at Runtime</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a> <a href="#fnref2:6" class="footnote-backref">↩︎</a> <a href="#fnref2:7" class="footnote-backref">↩︎</a> <a href="#fnref2:8" class="footnote-backref">↩︎</a> <a href="#fnref2:9" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 5 <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join at Runtime</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://www.waitingforcode.com/apache-spark-sql/whats-new-apache-spark-3-dynamic-partition-pruning/read" target="_blank" rel="noreferrer">What&#39;s New in Apache Spark 3: Dynamic Partition Pruning</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li></ol></section>',23)])])}const g=a(i,[["render",f]]);export{m as __pageData,g as default};

package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const n="/docs/assets/aqe-loop.IwQSATHw.svg",r="/docs/assets/aqe-loop.dark.DGbaxqJE.svg",m=JSON.parse('{"title":"Adaptive Query Execution","description":"","frontmatter":{"title":"Adaptive Query Execution"},"headers":[],"relativePath":"tuning-reference/aqe.md","filePath":"tuning-reference/aqe.md"}'),i={name:"tuning-reference/aqe.md"};function f(l,e,c,h,d,p){return t(),s("div",null,[...e[0]||(e[0]=[o("",23)])])}const g=a(i,[["render",f]]);export{m as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const r="/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg",i="/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg",b=JSON.parse('{"title":"Broadcast Sizing","description":"","frontmatter":{"title":"Broadcast Sizing"},"headers":[],"relativePath":"tuning-reference/bottleneck-broadcast-sizing.md","filePath":"tuning-reference/bottleneck-broadcast-sizing.md"}'),n={name:"tuning-reference/bottleneck-broadcast-sizing.md"};function d(l,e,c,h,f,p){return t(),o("div",null,[...e[0]||(e[0]=[s('<h1 id="bottleneck-broadcast-sizing" tabindex="-1">Broadcast Sizing <a class="header-anchor" href="#bottleneck-broadcast-sizing" aria-label="Permalink to &quot;Broadcast Sizing {#bottleneck-broadcast-sizing}&quot;">​</a></h1><p><span class="tag">BROADCASTSIZING</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A join has two ways to move data. The <a href="./joins.html">shuffle join</a> forces all-to-all traffic across the cluster; the broadcast join ships one side out to every executor and skips the <a href="./shuffle.html">shuffle</a> entirely. <code>spark.sql.autoBroadcastJoinThreshold</code> is the size gate that picks between them: it sets the largest table, in bytes, that Spark will broadcast when planning a join, and it defaults to 10 MB.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> Set it to <code>-1</code> and automatic broadcasting is off, so Spark always shuffles.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><img class="light-only" src="'+r+'" alt="How the small side estimate against the broadcast threshold picks between broadcasting a copy to every executor for a local join and an all-to-all shuffle join, with the broadcast path&#39;s driver collect and executor memory risks noted."><img class="dark-only" src="'+i+'" alt="How the small side estimate against the broadcast threshold picks between broadcasting a copy to every executor for a local join and an all-to-all shuffle join, with the broadcast path&#39;s driver collect and executor memory risks noted."><p>Given a join&#39;s estimated table sizes and that gate, there are two ways the choice can go wrong:</p><ul><li><strong>underBroadcast</strong>: one side is small enough to broadcast but is being shuffled anyway. The join pays for an all-to-all exchange it could have avoided.</li><li><strong>overBroadcast</strong>: a side that is too large is being broadcast, which risks running the driver or the executors out of memory.</li></ul><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Each join carries an estimated size for the side in question, which is compared against the broadcast threshold:</p><table tabindex="0"><thead><tr><th>Finding</th><th>Condition</th><th>What it means</th></tr></thead><tbody><tr><td><code>underBroadcast</code></td><td>small side is being shuffled instead of broadcast</td><td>a cheap broadcast join was left on the table</td></tr><tr><td><code>overBroadcast</code></td><td>an oversized side is being broadcast</td><td>the broadcast is a memory hazard</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Broadcasting the small side sends a full copy of that table to every executor. It pays off only when network bandwidth and executor memory are not the bottleneck; there, it is far cheaper than the shuffle it replaces.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup> That is the cost an <code>underBroadcast</code> finding is pointing at: a small dimension going through an all-to-all exchange when a copy on each executor would have run fast.</p><p><code>overBroadcast</code> is the opposite mistake, and it fails hard. Pushing a table that is too large through the broadcast path breaks in two ways: too-large task result errors and out-of-memory failures, both because the oversized table has to be collected on the driver and shipped whole to every executor.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup> The size gate exists for exactly this reason: a copy on every executor is harmless for a small table and a memory hazard for a large one.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>For <code>underBroadcast</code>, let the small side broadcast. If its estimated size sits above the 10 MB default but still fits comfortably in executor memory, raise <code>spark.sql.autoBroadcastJoinThreshold</code> to cover it, or force the strategy with <code>broadcast(df)</code> on the small side.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup></p><p>For <code>overBroadcast</code>, keep the large side out of the broadcast path. Lower the threshold so the oversized table no longer qualifies, or drop the explicit <code>broadcast()</code> hint if one is forcing it, and let Spark fall back to a shuffle join.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>Validated for both branches. <code>underBroadcast</code> and <code>overBroadcast</code> rest on the same well-documented mechanism: the broadcast threshold and its two failure modes are standard Spark behavior, so neither branch is experimental.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>The comparison is only as good as the size estimate behind it. Table sizes come from statistics that may be stale or missing, so a join that looks mis-sized can be measuring the wrong number. A large broadcast can also be deliberate: an operator who has sized executor memory for it may broadcast a table well past the default on purpose.</p><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 6 <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li></ol></section>',23)])])}const u=a(n,[["render",d]]);export{b as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const r="/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg",i="/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg",b=JSON.parse('{"title":"Broadcast Sizing","description":"","frontmatter":{"title":"Broadcast Sizing"},"headers":[],"relativePath":"tuning-reference/bottleneck-broadcast-sizing.md","filePath":"tuning-reference/bottleneck-broadcast-sizing.md"}'),n={name:"tuning-reference/bottleneck-broadcast-sizing.md"};function d(l,e,c,h,f,p){return t(),o("div",null,[...e[0]||(e[0]=[s("",23)])])}const u=a(n,[["render",d]]);export{b as __pageData,u as default};