sparkforensics-server 0.5.2 → 0.8.1

package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CHHgywoP.js

@@ -0,0 +1 @@
+import{_ as o,o as t,c as d,a5 as c}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Detector contract","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract.md","filePath":"contributor-guide/architecture/detector-contract.md"}'),s={name:"contributor-guide/architecture/detector-contract.md"};function a(r,e,n,i,h,l){return t(),d("div",null,[...e[0]||(e[0]=[c('<h1 id="detector-contract" tabindex="-1">Detector contract <a class="header-anchor" href="#detector-contract" aria-label="Permalink to &quot;Detector contract&quot;">​</a></h1><p><code>packages/core/src/detectors.ts</code> is the single source of Spark-optimization logic: one declarative <code>DETECTORS</code> entry per pattern, each carrying <code>type</code>, <code>scope</code> (<code>stage</code> / <code>app</code> / <code>config</code> / <code>sql</code>), <code>order</code>, <code>fixEffort</code>, <code>version</code>, a <code>thresholds</code> object, an <code>emits</code> list, an optional <code>docAnchor</code> (every entry but <code>incompleteRun</code> sets one), optional <code>inScorecard</code>, <code>property</code> and <code>suppressedBy</code>, and co-located <code>detect()</code> and <code>estimate()</code> functions. Each finding&#39;s <code>impactBand</code> and recommendation copy are set inside <code>detect()</code>, and so is its <code>remediation</code>: the structured <code>{kind: &#39;conf&#39;, key, direction, suggested}</code> form of the Spark properties that copy names (<code>increaseConf</code>, <code>decreaseConf</code> and <code>setConf</code> in <code>packages/core/src/remediation.ts</code>). A detector adds an entry only for a property its recommendation already names, and leaves <code>suggested</code> null unless it computes the value itself. Entries apply together unless the recommendation words them as alternatives (&quot;either ... or&quot;): <code>idleCapacityFix</code> in <code>detectors.ts</code> gives an idle-capacity finding with dynamic allocation off the alternatives <code>set spark.dynamicAllocation.enabled</code> and <code>decrease spark.executor.instances</code>, and with it on <code>decrease spark.dynamicAllocation.maxExecutors</code> plus <code>decrease spark.dynamicAllocation.minExecutors</code> when the logged floor is above 0. <code>Remediation</code> is a union: <code>codeFix(hint)</code> builds the <code>{kind: &#39;code&#39;, hint}</code> entry for a fix no property makes, which the skew family emits when its origin is <code>other</code> or skew-join handling is on and <code>aqe-skew.ts</code> finds the job&#39;s code is what has to change. Code that reads <code>key</code> checks <code>kind</code> first.</p><p>Each entry is built by the helper for its scope: <code>defineStageDetector</code>, <code>defineSqlDetector</code>, <code>defineAppDetector</code> or <code>defineConfigDetector</code>. The helper sets <code>scope</code>, infers the thresholds type from the entry&#39;s own <code>thresholds</code> literal, and types <code>detect</code> as a function property, so its parameters are checked strictly rather than bivariantly:</p><table tabindex="0"><thead><tr><th>Scope</th><th><code>detect</code> signature</th></tr></thead><tbody><tr><td><code>stage</code></td><td><code>detect(stage, ctx, thresholds)</code></td></tr><tr><td><code>sql</code></td><td><code>detect(sqlExec, ctx, thresholds)</code></td></tr><tr><td><code>app</code></td><td><code>detect(ctx, thresholds)</code></td></tr><tr><td><code>config</code></td><td><code>detect(target, thresholds)</code>, where <code>target</code> is <code>{ app }</code></td></tr></tbody></table><p><code>ctx</code> (<code>DetectorCtx</code>) is required, and its <code>app</code> is nullable as on <code>AppModel.app</code>, so a detector that reads the app without a guard, reads a threshold its entry doesn&#39;t declare, or expects another scope&#39;s target fails to compile. <code>detect</code> never reads <code>this</code>. The helper freezes the entry&#39;s <code>thresholds</code> and adds <code>withThresholds(overrides?)</code>, which returns <code>detect</code> with the thresholds bound: the entry&#39;s own, or the caller&#39;s overrides merged over them (see <a href="#tuning-thresholds">Tuning thresholds</a>). Runners such as <code>analyze()</code> call only that.</p><p><code>estimate(finding, ctx)</code> prices one of the entry&#39;s own findings: <code>finding</code> is typed as the <code>Finding</code> member of a type the entry <code>emits</code>, and <code>ctx</code> (<code>EstimateCtx</code>: <code>stages</code>, <code>occupancy</code>, <code>totalCores</code>) is the one occupancy sweep <code>analyze()</code> builds per run. It returns an <code>ImpactEstimate</code> or null (see <a href="./impact-estimation.html">Impact estimation</a>). Every entry must declare one; an entry with no waste model passes <code>noWasteModel</code>. <code>estimateImpact()</code> runs it after suppression, keyed by emitted type. The same <code>EstimateCtx</code> reaches <code>detect()</code> as <code>ctx.impact</code>, so a runtime floor can gate on the estimate the finding will display: <code>skew</code> and <code>straggler</code> build their tail claim once (<code>skewTailClaim</code>/<code>stragglerTailClaim</code>) and both their floor and their <code>estimate()</code> read it through <code>tailClaimImpact</code>. Shared model constants and helpers live in <code>packages/core/src/impact-model.ts</code>.</p><p><code>DETECTORS</code> is declared <code>as const satisfies readonly Detector[]</code>, so each entry keeps its literal <code>type</code> and <code>emits</code>. Two unions derive from it: <code>DetectorType</code> (every entry&#39;s own <code>type</code>) and <code>FindingType</code> (every type an entry&#39;s <code>emits</code> lists, the finding types that actually appear on findings). <code>emits</code> is <code>[type]</code> for every entry except <code>broadcastSizing</code>, whose one plan walk emits <code>overBroadcast</code> and <code>underBroadcast</code> and never its own name. Every per-type lookup keys on the emitted <code>FindingType</code>, never on <code>DetectorType</code>: code that needs the emitted types of an entry reads its <code>emits</code> list. Code that iterates entries generically, such as <code>analyze()</code>, reads them through the <code>Detector</code> type, a union over scopes with the thresholds type erased, <code>detect</code> left off and <code>estimate</code> taking any <code>Finding</code>, so switching on <code>scope</code> narrows the bound function <code>withThresholds()</code> returns.</p><p>How a finding type is presented is registered once, in <code>FINDING_PRESENTATION</code> (<code>packages/core/src/finding-presentation.ts</code>), typed <code>{ [T in FindingType]: FindingPresentation&lt;T&gt; }</code> so the compiler requires exactly one row per emitted type. A row holds the type&#39;s <code>name</code>, board <code>tag</code>, <code>actionLabel(finding)</code>, <code>genericRecommendation(finding)</code> and <code>thresholdSummary(thresholds)</code>, each finding argument typed as that type&#39;s <code>Finding</code> member and <code>thresholds</code> as the emitting entry&#39;s own <code>thresholds</code>. <code>FINDING_NAMES</code>, <code>TYPE_TAG_MAP</code>, <code>getThresholdSummary</code>, <code>findingActionLabel</code> and <code>coreFindingGenericRecommendation</code> all read it. The table sits beside <code>DETECTORS</code> rather than on its entries because the HTML export renders names, tags and labels but may not reach <code>detectors.ts</code> (see <a href="./state-and-history.html#run-interpretation">Run interpretation</a>); its import from <code>detectors.ts</code> is type-only. A detector&#39;s <code>scope</code> and <code>order</code> reach renderers through <code>detectorInfoByType()</code> (<code>detector-docs.ts</code>), keyed by emitted type, which the run interpretation ships; the Alerts clean-check grouping reads <code>detectors[type].scope</code>.</p><p>Each finding type has its own shape in <code>packages/core/src/finding-types.ts</code>: <code>Finding</code> is a union discriminated on <code>type</code>, and a compile-time check in <code>detectors.ts</code> fails when its members and <code>FindingType</code> differ. Each member splits into a <code>&lt;Type&gt;Evidence</code> interface, the fields the evidence report publishes (listed again in <code>EVIDENCE_KEYS</code> in <code>evidence-report.ts</code>, checked both ways), and fields declared only on <code>&lt;Type&gt;Finding</code>, which other core modules read but the report never publishes. <code>value</code> is always a magnitude; a text-valued finding (<code>stageFailed</code>, <code>configAudit</code>, <code>incompleteRun</code>) sets <code>valueText</code> instead. So a new detector type needs an <code>emits</code> entry, an <code>estimate()</code> on that entry that covers it, a <code>finding-types.ts</code> member, an <code>EVIDENCE_KEYS</code> entry, an <code>ID_DISCRIMINATORS</code> entry in <code>analyzer.ts</code>, a <code>FINDING_PRESENTATION</code> row and a view <code>REGISTRY</code> entry. The compiler reports each one that is missing. Tests and process, not the compiler, cover the rest: a new board tag goes in AGENTS.md&#39;s tag list (<code>packages/core/test/tag-vocabulary.test.js</code> fails otherwise), an intended finding change is checked with <code>node dev/bench-analyze.mjs --check dev/corpus-snapshot.json</code> and then <code>--update</code>d, and the PR needs a <code>.changeset/*.md</code>. A <code>docAnchor</code> must have a section in the detection docs (<code>packages/core/test/docs-config.test.js</code>). The view narrows with <code>findingsOfType(catalog, type)</code> (<code>packages/core/src/findings-of-type.ts</code>) rather than re-declaring a finding&#39;s fields.</p><p>Both consumers are thin loops over that array:</p><ul><li><p><code>packages/core/src/analyzer.ts</code>: <code>analyze()</code> runs every entry regardless of scope, skipping only <code>inScorecard:false</code> ones, then applies <code>suppressedBy</code> (below); <code>auditConfig()</code> separately runs the <code>scope:&#39;config&#39;</code> entries. The three <code>configAudit</code> entries stay out of the bottleneck catalog because each sets <code>inScorecard:false</code>, not because of <code>scope:&#39;config&#39;</code>: a config-scope detector without that flag would run through <code>analyze()</code> too. Each finding is stamped with its entry&#39;s <code>docAnchor</code>.</p></li><li><p><code>src/view/detector-registry.tsx</code>: <code>REGISTRY</code> maps each emitted finding type to <code>{component, region, widgetId, routeable}</code> (view-only concerns), checked with <code>satisfies Record&lt;FindingType, RegistryEntry&gt;</code>. Each type has its own component, and no two entries share one. The executor-count chart (<code>ExecutorCountChart</code>) is a <code>ReferenceSection</code> tile, not a <code>REGISTRY</code> entry. <code>orderedWidgets(detectors)</code> takes the run interpretation&#39;s <code>detectorInfoByType()</code> record (<code>DetectorInfo</code> per type), keeps the types that have a <code>REGISTRY</code> entry, and sorts <code>action</code>-region components before <code>reference</code>-region ones, then by ascending <code>order</code>, using the record&#39;s declaration order to break ties.</p><p>Each widget component receives the full catalog and self-gates when it has nothing to show, rendering <code>null</code> or a muted &quot;no issue&quot; card for the always-visible ones. <code>orderedWidgets()</code> itself has no empty/non-empty branching, since it iterates the static per-type detector info, not the runtime <code>catalog</code>.</p></li></ul><p>Default thresholds live only in each entry&#39;s <code>thresholds</code>; see <a href="./detector-contract/threshold-tables.html#bottleneck-thresholds">Bottleneck thresholds</a>. Only the CLI and the MCP server can override them, per run: see <a href="#tuning-thresholds">Tuning thresholds</a>.</p><h2 id="confidence-disclosure" tabindex="-1">Confidence disclosure <a class="header-anchor" href="#confidence-disclosure" aria-label="Permalink to &quot;Confidence disclosure&quot;">​</a></h2><p>A <code>Finding</code> may carry <code>confidence: &#39;low&#39; | &#39;medium&#39; | &#39;high&#39;</code> plus a <code>validationRequired</code> string. <code>RowStatusCluster</code> (<code>src/view/RowStatusCluster.tsx</code>) renders it on widget rows (the verdict&#39;s steps print their own &quot;verify before acting&quot; line), gated to Advanced density: a plain &quot;&lt;confidence&gt; confidence&quot; badge whose tooltip carries the full <code>validationRequired</code> text. A finding with no <code>confidence</code> field renders identically to a fully-validated one, so every detector whose thresholds are our own unvalidated noise floor (listed in <a href="./detector-contract/threshold-tables.html#threshold-sources">Threshold sources</a>) should set both fields, not just the ones that happen to already have <code>RowStatusCluster</code> wired into their widget. <code>skew</code>, <code>straggler</code>, and <code>gc</code> set <code>confidence</code> for exactly this reason: their runtime-floor thresholds carry the same kind of unvalidated-noise-floor caveat <code>coreLocality</code>, <code>autoscalingChurn</code>, and <code>memoryUtilization</code>&#39;s <code>wasteModel</code> variant already disclose. None of these hardcode a single confidence value: each scales <code>&#39;low&#39; | &#39;medium&#39; | &#39;high&#39;</code> off how far the finding sits past its own detector&#39;s threshold, via a small named helper placed just above the <code>DETECTORS</code> array (e.g. <code>skewConfidence</code>, <code>coreLocalityConfidence</code>, <code>cachingReuseConfidence</code>) rather than an inline literal.</p><h2 id="the-fixeffort-field" tabindex="-1">The <code>fixEffort</code> field <a class="header-anchor" href="#the-fixeffort-field" aria-label="Permalink to &quot;The `fixEffort` field&quot;">​</a></h2><p>Each <code>Detector</code> entry also carries <code>fixEffort: &#39;config&#39; | &#39;code&#39; | &#39;rearchitect&#39;</code>, alongside <code>order</code> and <code>thresholds</code>: a rough estimate of how much work resolving the finding takes.</p><p>No view reads it: the ranking <code>FixTheseFirst</code> (<code>src/view/widgets/FixTheseFirst.tsx</code>) renders is computed in core by <code>interpretRun</code> (<code>rankedRollup</code>/<code>rankFindings</code> in <code>packages/core/src/recommendation-rollup.ts</code>), by estimate tier and impact, never by <code>fixEffort</code>. See <a href="./widget-rendering.html#widget-rendering-order">Widget rendering order</a> for how it ranks findings.</p><p>Two shared helpers back multiple detectors and reports. <code>packages/core/src/plan-tree-walk.ts</code>&#39;s <code>walkPlanTree(root, visit, {dedupe})</code> is the iterative pre-order plan-tree traversal used by <code>detectors.ts</code>, <code>plan-summary.ts</code>, <code>plan-duration-attribution.ts</code>, <code>plan-dot.ts</code> and <code>plan-graph-model.ts</code>. <code>packages/core/src/core-count.ts</code> holds the shared core-count logic. Its <code>computePeakConcurrentCores</code>/<code>computePeakConcurrentExecutorCount</code> sweeps back <code>detectors.ts</code>&#39;s <code>efficiency-model.ts</code>&#39;s zero-skew floor and the cluster size the <code>utilization</code> finding reports. The idle capacity the <code>utilization</code> and <code>memoryUtilization</code> entries, <code>efficiency-model.ts</code> and <code>wasted-core-hours.ts</code> measure against is the allocation (<code>allocatedCoreMs()</code> in <code>allocation.ts</code>, cores × time each executor was alive), so the Scorecard&#39;s Unused core time and the verdict&#39;s idle figure share one capacity that never exceeds what the run held. <code>computeTotalCores(app, executorsAdded)</code>, used only by <code>scaling-sim.ts</code>, sums every <code>ExecutorAdded</code> event with no regard for overlap, so under executor churn (spot preemption, <code>dynamicAllocation</code> replacement) it double-counts a churned executor&#39;s capacity against its replacement&#39;s; the peak-concurrent sweeps don&#39;t.</p><h2 id="effective-configuration" tabindex="-1">Effective configuration <a class="header-anchor" href="#effective-configuration" aria-label="Permalink to &quot;Effective configuration&quot;">​</a></h2><p>A detector that reads or suggests a Spark property resolves it through <code>effectiveSparkConf()</code> (<code>packages/core/src/spark-conf.ts</code>), which layers three sources: the SQL execution&#39;s <code>modifiedConfigs</code> (the session settings that differ from the SparkContext&#39;s, as <code>spark.conf.set</code> leaves them), the app&#39;s Spark Properties, then Spark&#39;s default for the run&#39;s version. It returns the value and the layer it came from (<code>query</code>, <code>app</code> or <code>default</code>). <code>sparkConfDefault()</code> is the version table alone; its entries are read from Spark&#39;s <code>SQLConf.scala</code> and <code>config/package.scala</code> at the release tags, and a property whose default depends on the cluster or on another property has no entry. Add a property to the table only with the release that introduced each of its defaults, and a per-version test.</p><p>A detector scoped to one SQL execution (<code>skew</code>, <code>straggler</code>, <code>shuffle</code>, <code>partitionSizing</code>, <code>underBroadcast</code>, <code>overBroadcast</code>, <code>pythonUdf</code>) passes that execution&#39;s <code>modifiedConfigs</code>: <code>detectors.ts</code>&#39;s <code>stageApp()</code> and <code>queryApp()</code> return the app with them applied, so the helpers that take an app (<code>effectiveConf</code>, <code>loggedAs</code>, <code>switchFix</code>) need no second argument. Run-wide detectors use the app as logged. The frozen detection thresholds do not follow a per-query setting; only the advice does.</p><p>The skew family&#39;s advice for a stage that reads a shuffle feeding a join comes from <code>diagnoseJoinSkew</code> in <code>aqe-skew.ts</code>, which walks the execution&#39;s final plan against the conditions Spark&#39;s <code>OptimizeSkewedJoin</code> checks (join shape, join type, thresholds, extra shuffle) and reads the thresholds through the same effective conf. It returns <code>null</code> when the plan cannot say, and the detector then keeps the general advice. A new case is a new <code>AqeSkewCase</code>, a row in <code>finding-presentation.ts</code>&#39;s generic lines and a row in the user guide&#39;s <code>SKEW</code> table.</p><h2 id="finding-ids" tabindex="-1">Finding ids <a class="header-anchor" href="#finding-ids" aria-label="Permalink to &quot;Finding ids&quot;">​</a></h2><p>A finding&#39;s <code>id</code> is a stable hash of its <code>type</code>, its location (stage, SQL execution or audited config property), <code>metric</code>, <code>value</code> and the per-type discriminator fields in <code>ID_DISCRIMINATORS</code> (<code>packages/core/src/analyzer.ts</code>): the fields that tell apart sibling findings sharing a location and metric. Reordering or renaming a discriminator slot changes every id and needs an <code>EVIDENCE_SCHEMA_VERSION</code> bump.</p><h2 id="cross-detector-suppression" tabindex="-1">Cross-detector suppression <a class="header-anchor" href="#cross-detector-suppression" aria-label="Permalink to &quot;Cross-detector suppression&quot;">​</a></h2><p>An entry may name another entry&#39;s <code>type</code> in <code>suppressedBy</code>. Once every detector has run, <code>analyze()</code>&#39;s <code>applySuppression()</code> drops each of that entry&#39;s findings on a stage where the named detector emitted a finding. It reads the unsuppressed findings, so neither the two entries&#39; declaration order nor the order suppressions apply in changes the result, and <code>order</code> stays a display field only. A compile-time check in <code>detectors.ts</code> (<code>SuppressorsAreDetectors</code>) fails when <code>suppressedBy</code> names no entry.</p><p><code>stageSlowness</code> sets <code>suppressedBy: &#39;slowHost&#39;</code>: a stage <code>slowHost</code> already explains needs no generic &quot;this stage is slow&quot; finding. Suppression follows what <code>slowHost</code> actually emitted, so a run whose <code>slowHost</code> thresholds are tuned so it can&#39;t fire gets its <code>stageSlowness</code> findings back. The mechanism is deliberately minimal (same-stage, one named suppressor per entry), not a general dependency graph. <code>auditConfig()</code> doesn&#39;t apply it, since no <code>scope:&#39;config&#39;</code> entry sets <code>suppressedBy</code>.</p><h2 id="tuning-thresholds" tabindex="-1">Tuning thresholds <a class="header-anchor" href="#tuning-thresholds" aria-label="Permalink to &quot;Tuning thresholds&quot;">​</a></h2><p><code>analyze()</code>&#39;s eighth argument is <code>{ thresholds?: ThresholdOverrides }</code>: per-entry overrides keyed by entry <code>type</code>, each a partial of that entry&#39;s own <code>thresholds</code>. Omitted, every entry runs its defaults; the dashboard never passes it. The CLI&#39;s and the MCP server&#39;s <code>--thresholds &lt;file&gt;</code> read a JSON file of that shape (<code>packages/core/src/cli/threshold-config.ts</code>) and validate it with <code>parseThresholdOverrides()</code> (<code>packages/core/src/threshold-overrides.ts</code>), which refuses an unknown detector or threshold, a negative or non-numeric value, a tier table of a different length or out of ascending order, an entry with no thresholds (<code>stageFailed</code>, <code>incompleteRun</code>), and any <code>configAudit</code> (config-scope) override: those checks compare against Spark&#39;s own defaults, so there is nothing to tune. A file that can&#39;t be read or parsed refuses the run the same way. The <a href="./../../user-guide/getting-started/tuning-thresholds.html#tuning-detector-thresholds">user guide</a> documents the file.</p><p>A finding from an entry whose overrides move a threshold off its default carries <code>tunedThresholds</code> (<code>{ &lt;name&gt;: { value, default } }</code>), and once its estimate is attached the analyzer appends a caveat to its <code>validationRequired</code> naming the tuned values. When the finding has an estimate figure (wall-clock or raw waste), the caveat adds that impact estimates are calibrated against the default thresholds (see <a href="./impact-estimation.html">Impact estimation</a>), so its estimate is unvalidated; an informational finding gets the label alone. An override equal to the default labels nothing. Tuning a <code>suppressedBy</code> target changes which of the suppressed entry&#39;s findings survive, so those findings carry the suppressor&#39;s tuned thresholds too, named <code>&lt;suppressor&gt;.&lt;name&gt;</code> (e.g. <code>slowHost.minHosts</code> on <code>stageSlowness</code>). <code>straggler</code> has no <code>suppressedBy</code> but judges a tail with <code>skew</code>&#39;s resolved thresholds (<code>DetectorCtx.skewThresholds</code>), so its findings carry <code>skew</code>&#39;s tuned thresholds the same way (<code>THRESHOLD_DEPENDENCY</code> in <code>threshold-overrides.ts</code>). Only that one link is followed. The evidence report repeats the label on the finding row, the clean check, the <code>detectors</code> catalog row (whose <code>thresholds</code> are then the effective ones) and in <code>summary.tunedThresholds</code>; see <a href="./worker-protocol/evidence-report.html#portable-evidence-report">Portable evidence report</a>.</p><p>A tuned <code>floorPctWarn</code>/<code>floorPctCrit</code> on <code>skew</code> or <code>straggler</code> also grades that entry&#39;s own findings in <code>deriveImpactBand</code> (<code>impact-band.ts</code>); every other finding keeps the run-wide default floors. <code>--max-skew</code> recomputes the ratio with the run&#39;s effective <code>minTasksForP95</code>, so the budget measures the same ratio the skew finding reports. Caveat text that names a threshold (<code>gc</code>, <code>skew</code>, <code>straggler</code>, <code>memoryUtilization</code>, <code>coreLocality</code>) and the <code>broadcastSizing</code> over-broadcast recommendation state the value the detector ran with. The HTML export still renders the default-threshold analysis.</p>',31)])])}const f=o(s,[["render",a]]);export{u as __pageData,f as default};

package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js → contributor-guide_architecture_detector-contract.md.CHHgywoP.lean.js}

@@ -1 +1 @@
-import{_ as o,o as t,c as d,a5 as c}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Detector contract","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract.md","filePath":"contributor-guide/architecture/detector-contract.md"}'),s={name:"contributor-guide/architecture/detector-contract.md"};function r(a,e,n,i,h,l){return t(),d("div",null,[...e[0]||(e[0]=[c("",25)])])}const g=o(s,[["render",r]]);export{u as __pageData,g as default};
+import{_ as o,o as t,c as d,a5 as c}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Detector contract","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract.md","filePath":"contributor-guide/architecture/detector-contract.md"}'),s={name:"contributor-guide/architecture/detector-contract.md"};function a(r,e,n,i,h,l){return t(),d("div",null,[...e[0]||(e[0]=[c("",31)])])}const f=o(s,[["render",a]]);export{u as __pageData,f as default};

package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Plan attribution","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract/plan-attribution.md","filePath":"contributor-guide/architecture/detector-contract/plan-attribution.md"}'),n={name:"contributor-guide/architecture/detector-contract/plan-attribution.md"};function c(d,e,r,i,l,u){return o(),a("div",null,[...e[0]||(e[0]=[s('<h1 id="plan-attribution" tabindex="-1">Plan attribution <a class="header-anchor" href="#plan-attribution" aria-label="Permalink to &quot;Plan attribution&quot;">​</a></h1><p>How plan-derived findings are attributed to operators and to stages.</p><h2 id="per-operator-duration-attribution" tabindex="-1">Per-operator duration attribution <a class="header-anchor" href="#per-operator-duration-attribution" aria-label="Permalink to &quot;Per-operator duration attribution&quot;">​</a></h2><p><code>packages/core/src/plan-duration-attribution.ts</code> (entry <code>attributeStageDurationToPlan(planTree, stagesById, sqlExec)</code>) approximates how a SQL execution&#39;s stage wall-time splits across plan operators, returning a <code>Map&lt;planNode, milliseconds&gt;</code>. It cuts the plan tree at Exchange boundaries into connected components: because <code>resolvePlanTree</code> in <code>event-handlers.ts</code> always synthesizes a <code>read</code> node wrapping a <code>write</code> node for every raw <code>Exchange</code>/<code>BroadcastExchange</code>, the cut is keyed off <code>PlanNode.exchangeRole === &#39;read&#39;</code> on the parent, not a name regex: the write half starts the new component, the read half stays in its parent&#39;s. A node with no <code>exchangeRole</code> at all (for example <code>ReusedExchange</code>, which is never split) never starts a new component on its own. Each component receives a stable pre-order identity and separate parent/depth/traversal metadata; the identity itself does not encode its count of Exchange ancestors. It zips components deepest-first by that explicit depth against submission-ordered stage IDs, then apportions each matched stage&#39;s wall-time across that component&#39;s nodes by timing-metric weight, falling back to an even split when no node carries a timing metric.</p><p>This is best-effort inference, not measurement. Spark&#39;s event model exposes no ground truth for per-operator time within a stage; the Exchange-boundary segmentation and deepest-component-to-earliest-stage zip are heuristics. Treat the per-operator numbers as directional hints, never as authoritative timings, and do not build hard thresholds or findings on top of them.</p><h2 id="linking-a-sql-execution-to-its-stages" tabindex="-1">Linking a SQL execution to its stages <a class="header-anchor" href="#linking-a-sql-execution-to-its-stages" aria-label="Permalink to &quot;Linking a SQL execution to its stages&quot;">​</a></h2><p><code>sql.get(id).stageIds</code> is always empty: the parser never populates it. Every <code>scope: &#39;sql&#39;</code> detector and the plan views derive the linkage from each stage&#39;s own <code>sqlExecutionId</code> with <code>stageIdsForSqlExec</code> (<code>sql-stages.ts</code>).</p><h2 id="stage-id-attribution-for-plan-advisor-findings" tabindex="-1">Stage-ID attribution for Plan Advisor findings <a class="header-anchor" href="#stage-id-attribution-for-plan-advisor-findings" aria-label="Permalink to &quot;Stage-ID attribution for Plan Advisor findings&quot;">​</a></h2><p>The Plan Advisor detectors (<code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>nestedLoopJoin</code>, <code>pythonUdf</code>, <code>broadcastSizing</code> in <code>packages/core/src/detectors.ts</code>) each attribute their finding to a narrowed <code>stageIds</code> set rather than the whole SQL execution: <code>PlanNode.stageIds</code> is resolved once per plan tree at parse time by unioning, per node, every metric&#39;s accumulator ID against a <code>taskAccumStages: Map&lt;accumulatorId, Set&lt;stageId&gt;&gt;</code> built while parsing <code>TaskEnd</code> events, then clipping the result to the execution&#39;s own stage set. An accumulator ID occasionally points to a <em>different</em> execution&#39;s stages, e.g. a <code>ReusedSubquery</code> computed once and reused verbatim, and the clip prevents misattributing that other execution&#39;s work. Each detector unions its implicated node(s)&#39; <code>stageIds</code> and falls back to the execution-wide set only when no implicated node has any coverage; a finding never partially blends a narrowed set with the execution-wide one. <code>nestedLoopJoin</code> is the exception to the fallback: it claims the time of the stages that run the join, so a join node with no stage coverage gets an empty <code>stageIds</code> and no time claim rather than the whole execution&#39;s stages. When an execution has no stage universe at all (no jobs ever recorded against it, which is true for 42% of real-log SQL executions with a plan tree, typically job-less/driver-only executions), the clip drops every candidate stage ID instead of passing them through: every node in that execution&#39;s tree ends up with no <code>stageIds</code> anywhere, same &quot;coverage is partial&quot; framing as below. (The clip is unconditional on <code>executionStageIds</code> being present: treating &quot;no stage universe&quot; as &quot;no clip&quot; would let a foreign accumulator ID collision, e.g. the <code>ReusedSubquery</code> case above, leak another execution&#39;s stages into a job-less execution&#39;s nodes.)</p><p>Coverage is partial by Spark&#39;s own design: whole-stage-codegen wrapper nodes (<code>InputAdapter</code>, and other purely structural passthrough markers) carry no accumulators at all, and <code>BroadcastExchangeExec</code>&#39;s own metrics are computed entirely on the driver and never appear on any <code>TaskEnd</code> (real Spark behavior). Those driver-computed metrics live specifically on the synthesized <em>write</em> half (<code>exchangeRole: &#39;write&#39;</code>); the <em>read</em> half always carries <code>metrics: []</code>. The write half&#39;s immediate child, which does carry executor-side metrics, is unioned in instead, see <code>overBroadcast</code>&#39;s wiring. A <code>TaskEnd</code> arriving after its stage has already been finalized is also silently excluded from <code>taskAccumStages</code>, consistent with the parser&#39;s existing out-of-order tolerance elsewhere.</p><p><code>planTree</code> itself is kept current against Spark&#39;s adaptive query execution (AQE) re-plans: <code>SparkListenerSQLAdaptiveExecutionUpdate</code> events overwrite the execution&#39;s <code>sparkPlanInfo</code> last-write-wins, so accumulator-ID evidence is matched against the plan that actually ran rather than a stale pre-AQE snapshot. Since a superseded plan is never read, <code>dispatchLine</code> holds an open execution&#39;s latest update as unparsed text and parses only that one, when <code>SQLExecutionEnd</code> arrives or at parse completion (<code>deferAdaptiveUpdate</code>, <code>event-handlers.ts</code>). It recognizes an update from the flat first and last pieces <code>buildChunkDecoder</code> reports for a line joined across slices (<code>JoinedLine</code>), so a superseded update is never copied into one flat string either. The raw <code>sparkPlanInfo</code> stays worker-side and is released once <code>SQLExecutionEnd</code> resolves it into <code>planTree</code>; <code>physicalPlanDescription</code> (Spark&#39;s text rendering of the plan) is emptied before <code>JSON.parse</code> and never retained, except the one <code>Arguments:</code> line of a Delta write command, which <code>delta-targets.ts</code> reads to name the write target (<code>stripPlanDescription</code>, <code>event-handlers.ts</code>). <code>buildChunkDecoder</code> decodes each decompressed chunk in slices of at most 512 KiB, and drops the bytes of a value that crosses a slice boundary without decoding them.</p><p>A plan node&#39;s metric values come from two sources. Driver-side values (file counts, broadcast <code>data size</code>) arrive in <code>SparkListenerDriverAccumUpdates</code> and are summed per execution in <code>accumState</code>. Executor-side values (Exchange <code>data size</code>, <code>peak memory</code>, <code>spill size</code>, operator row counts, <code>data sent to Python workers</code>, timings) come from the <code>Accumulables</code> of each <code>SparkListenerStageCompleted</code> event, kept per execution by <code>recordStageSqlAccumulables</code> in <code>executorAccumState</code>. Only entries whose <code>Metadata</code> is <code>sql</code> are kept, and <code>Value</code> is a decimal string. A stage&#39;s value is the accumulator&#39;s running total across every stage (<code>DAGScheduler.updateAccumulators</code> merges each task&#39;s update into the driver-side accumulator and records its value), so a later stage replaces an earlier one&#39;s entry instead of adding to it. <code>resolvePlanTree</code> uses the driver-side value when there is one and the executor-side one otherwise, and marks the latter <code>executorSide: true</code>. <code>average</code> metrics get no executor-side value: their accumulator holds a sum of per-task averages, not the average the SQL tab shows. Plan-shape fingerprints (<code>computePlanShapes</code>, <code>findCompositeCandidates</code>) skip <code>executorSide</code> metrics, so <code>duplicatePlanSubtree</code> and <code>cachingOpportunity</code> ids do not depend on which tasks reported a value.</p><p>No eviction/pruning is added to <code>taskAccumStages</code>, a deliberate choice, not an oversight: measured on real logs, it holds roughly 1,050 keys per compressed MB (9,850 keys on an 11.6 MB fixture, about 29,000 keys on a 28.1 MB fixture). Extrapolated to a 240MB+ log, the scale this tool targets, that is roughly 250,000 keys, around 45 MB of heap for an equivalent synthetic <code>Map&lt;number, Set&lt;number&gt;&gt;</code>. This heap estimate is small relative to this tool&#39;s other in-memory state.</p><p>Per-execution pruning (e.g. dropping a <code>taskAccumStages</code> entry once its stage finalizes or its owning SQL execution resolves, mirroring how <code>accumState</code> is cleared in <code>endSqlExecution</code>) is deliberately not done either: unlike <code>accumState</code>, <code>taskAccumStages</code> is one global, un-scoped map read by every execution&#39;s <code>resolvePlanTree</code> call, and the <code>ReusedSubquery</code> case above depends on a stage recorded under one execution still being visible when a later execution resolves. Pruning on any single execution&#39;s lifecycle would break that cross-execution lookup. What is bounded is the growth from a single pathological event: <code>TaskEndEventSchema</code>&#39;s <code>Accumulables</code> array is capped at <code>MAX_ACCUMULABLES_PER_TASK</code> (10,000, <code>event-schemas.ts</code>), well above any real plan&#39;s per-task metric count, so a single crafted <code>TaskEnd</code> can&#39;t grow the map past that per-event bound; a <code>TaskEnd</code> exceeding it fails schema validation and the line is skipped (counted in <code>skippedLines</code>) like any other malformed event. Each accumulable is checked for its numeric <code>ID</code> only: <code>Update</code>/<code>Value</code> are unread, and Spark writes some as JSON arrays (<code>internal.metrics.updatedBlockStatuses</code>), which a stricter check would reject along with the whole task. When every entry has Spark&#39;s flat <code>{&quot;ID&quot;:n,...}</code> form, <code>parseTaskEnd</code> (<code>event-handlers.ts</code>) reads the IDs with a string scan and cuts the array out before <code>JSON.parse</code>; any other shape is parsed whole. On that path the IDs skip schema validation: <code>parseAndDispatch</code> passes them straight to <code>accumulateTask</code>, and a list of more than <code>MAX_ACCUMULABLES_PER_TASK</code> IDs is parsed whole so the schema still rejects it. <code>accumulateTask</code> also records a stage&#39;s ID list only when it differs from that stage&#39;s previous TaskEnd, since Spark gives a stage&#39;s tasks one accumulator set.</p>',14)])])}const g=t(n,[["render",c]]);export{p as __pageData,g as default};

package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.lean.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Plan attribution","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract/plan-attribution.md","filePath":"contributor-guide/architecture/detector-contract/plan-attribution.md"}'),n={name:"contributor-guide/architecture/detector-contract/plan-attribution.md"};function c(d,e,r,i,l,u){return o(),a("div",null,[...e[0]||(e[0]=[s("",14)])])}const g=t(n,[["render",c]]);export{p as __pageData,g as default};

package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.js

@@ -0,0 +1 @@
+import{_ as t,o,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Threshold tables","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract/threshold-tables.md","filePath":"contributor-guide/architecture/detector-contract/threshold-tables.md"}'),s={name:"contributor-guide/architecture/detector-contract/threshold-tables.md"};function r(i,e,c,n,l,h){return o(),d("div",null,[...e[0]||(e[0]=[a('<h1 id="threshold-tables" tabindex="-1">Threshold tables <a class="header-anchor" href="#threshold-tables" aria-label="Permalink to &quot;Threshold tables&quot;">​</a></h1><p>The threshold tables for every bottleneck detector, and where each finding&#39;s impact band comes from.</p><h2 id="bottleneck-thresholds" tabindex="-1">Bottleneck thresholds <a class="header-anchor" href="#bottleneck-thresholds" aria-label="Permalink to &quot;Bottleneck thresholds&quot;">​</a></h2><p>Every change to <code>packages/core/src/detectors.ts</code> should reference this table.</p><p>Every finding&#39;s <code>impactBand</code> comes from one of two places. For any finding whose <code>impactEstimate</code> carries a <code>wallClock</code> estimate (the common case for most rules below), <code>analyzer.ts</code> calls <code>deriveImpactBand()</code> (<code>packages/core/src/impact-band.ts</code>) immediately after <code>estimateImpact()</code>, which sets <code>.impactBand</code> purely from <code>wallClock.high</code> as a fraction of the app&#39;s total duration (<code>&gt;= 2%</code> critical, <code>&gt;= 0.5%</code> warning, else info: <code>IMPACT_FLOOR_PCT_CRIT</code>/ <code>IMPACT_FLOOR_PCT_WARN</code>, which <code>skew</code>&#39;s <code>floorPctWarn</code> and <code>straggler</code>&#39;s <code>floorPctWarn</code>/<code>floorPctCrit</code> default to, so a tail those gates admit grades at least warning; a tuned <code>skew</code> or <code>straggler</code> floor also grades that entry&#39;s own findings, see <a href="./../detector-contract.html#tuning-thresholds">Tuning thresholds</a>). For those rules, the table below documents their firing gate plus their fixed fallback constant, which surfaces only when this run&#39;s finding of that type didn&#39;t get a wallClock estimate (a stage excluded from the occupancy sweep). For rules whose finding type never gets a wallClock estimate (<code>resourceOnly</code>/<code>informational</code> basis, e.g. <code>configAudit</code>, or a rule that keeps its own ratio-tiered classification, e.g. <code>failures</code>), the full threshold table below is the real, displayed classification: <code>detectors.ts</code> sets <code>impactBand</code> directly and nothing overwrites it. <code>partitionSizing</code>&#39;s <code>maxPartitionTooBig</code> rule is a third case: it does carry a <code>wallClock</code> estimate but is explicitly exempted in <code>deriveImpactBand()</code> because it&#39;s a hardcoded-critical OOM/crash-risk safety signal, not a time-recovery one, so <code>detectors.ts</code>&#39;s own classification stands regardless of how small that estimate is relative to the run.</p><h3 id="fixed-fallback-only-usually-wallclock-derived-instead" tabindex="-1">Fixed fallback only (usually wallClock-derived instead) <a class="header-anchor" href="#fixed-fallback-only-usually-wallclock-derived-instead" aria-label="Permalink to &quot;Fixed fallback only (usually wallClock-derived instead)&quot;">​</a></h3><p>These rules have no <em>band</em> tiers in <code>packages/core/src/detectors.ts</code> (<code>deriveImpactBand</code> sets the band whenever a wallClock estimate is available); the constant in the last column is only a floor-case fallback. Their <em>firing</em> gate lives in each entry&#39;s <code>thresholds</code> object: it decides whether the rule reports anything, so it stays documented here in full.</p><table tabindex="0"><thead><tr><th>Rule</th><th>Fires when</th><th>Fallback</th></tr></thead><tbody><tr><td>Task skew</td><td><code>taskDurationP95 / taskDurationP50 &gt; 3×</code> (<code>taskDurationP50</code> and the other P50 fields are the textbook median: the mean of the two middle values on an even count; P95 stays nearest-rank) (<code>taskDurationMax / P50</code> for stages under <code>minTasksForP95</code> = 20 tasks), <strong>and</strong> the occupancy-clipped tail recovery (<code>tailRecoveryMs</code>: <code>finalizeStage</code>&#39;s task-level replay of the stage with every task over 4× P50 capped at P50) is ≥ <code>floorPctWarn</code> = 0.5% of app runtime. The clip floors the claim at the longest task the fix leaves, not the current one, and at the core work the fix leaves over every core (see impact-estimation.md&#39;s occupancy section); <code>straggler</code>&#39;s floors use the same clip, <strong>and</strong> the tail is data (<code>tailAttribution</code>: run time proportional to the slow tasks&#39; input plus shuffle-read bytes or records accounts for at least <code>dataShareMin</code> = 0.5 of their extra time). An <code>unattributed</code> tail (no data volume to compare and nothing else explains the largest share, or no tail attribution) is <code>straggler</code>&#39;s</td><td><code>warning</code></td></tr><tr><td>Shuffle read</td><td><code>shuffleReadBytes &gt; minBytes</code> = 50 MiB, on a stage that isn&#39;t shorter than <code>stageFloorPct</code> = 0.5% of the run (a zero-length stage or an unknown run duration passes; the claim is clipped to the stage, so a shorter one would grade <code>info</code>, the shuffle itself still there)</td><td><code>info</code></td></tr><tr><td>Partition sizing: skew</td><td><code>shuffleReadMax &gt; 5×</code> <code>shuffleReadP50</code> <strong>and</strong> <code>shuffleReadMax &gt; 256 MiB</code></td><td><code>warning</code></td></tr><tr><td>Partition sizing: low parallelism</td><td><code>shuffleReadBytes ≥ 1 GiB</code> <strong>and</strong> <code>taskCount ≤ 7</code></td><td><code>warning</code></td></tr><tr><td>Partition sizing: oversized partition</td><td><code>shuffleReadMax ≥ 5 GiB</code></td><td><code>critical</code></td></tr><tr><td>GC</td><td><code>executorRunTime ≥ minRunTimeMs</code> = 10 s <strong>and</strong> <code>gcPct &gt; 10%</code></td><td><code>warning</code></td></tr><tr><td>GC (low / cost)</td><td><code>executorRunTime ≥ 10 s</code> <strong>and</strong> <code>gcPct &lt; lowInfoPct100</code> = 5% (checked only when the GC row above did not fire <strong>and</strong> the log has no measured executor heap peak) <strong>and</strong> the stage lasts ≥ <code>lowInfoFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, such as one with no completion time). Gets no wall-clock estimate (an over-provisioning signal), so this band always stands. The floor drops low-GC notes from stages that barely ran: low GC is still true on those stages, but a memory-sizing note from a stage that barely ran adds nothing</td><td><code>info</code></td></tr><tr><td>Spill</td><td>any non-zero <code>memoryBytesSpilled</code>, on a stage that isn&#39;t shorter than <code>stageFloorPct</code> = 0.5% of the run (as for shuffle read: a shorter stage would grade <code>info</code>). The magnitude sub-table below classifies <em>how much</em>, but does not gate firing</td><td><code>warning</code></td></tr><tr><td>Cold start</td><td><code>firstExecutorAddedAt − firstStageSubmittedAt &gt; gapSeconds</code> = 30 s (no finding without executor-added events): the time a runnable stage waited for its first executor. An executor added before the first stage and still alive at submission means no wait and no finding; one removed at or before submission is ignored, so the gap runs to the next executor added after it.</td><td><code>warning</code></td></tr><tr><td>Slow host: mean-duration ratio</td><td>stage has ≥ <code>minHosts</code> = 3 hosts (or executors) and ≥ <code>minTasks</code> = 15 tasks, and lasts ≥ <code>stageFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, which gets no estimate and keeps its fallback band; every slow-host finding on a shorter stage would grade <code>info</code>, the imbalance itself still true; this gate covers every slowHost row); then per host: mean task duration / overall median ≥ <code>ratioWarn</code> = 2.0× <strong>and</strong> host task-share ≥ <code>minShare</code> = 20% <strong>and</strong> host mean ≥ <code>floorMs</code> = 1000 ms (absolute-magnitude floor, rules out sub-second noise)</td><td><code>warning</code></td></tr><tr><td>Slow host: duration-share</td><td>same stage gate as the row above; then per host: ≥ <code>shareWarn</code> = 75% of the stage&#39;s total task-duration <strong>and</strong> ≥ <code>taskShareWarn</code> = 50% of its task count</td><td><code>warning</code></td></tr><tr><td>Stage slowness: absolute fallback, suppressed when <code>slowHost</code> already fired</td><td>stage wall-clock duration ≥ <code>infoMin</code> = 15 min. The band then comes from the partitioning-headroom estimate (see impact-estimation.md), not the duration</td><td><code>info</code></td></tr><tr><td>Straggler / speculative-execution</td><td><code>taskCount ≥ minTasks</code> = 10, <strong>and</strong> the stage lasts ≥ <code>floorPctWarn</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, such as one with no completion time; a shorter stage&#39;s tail can&#39;t cost more than its own duration, so every finding there would grade <code>info</code>, the slow tail itself still true), <strong>and</strong> either any speculative task ran, <strong>or</strong> straggler share &gt; <code>shareWarn</code> = 5%, <strong>or</strong> straggler share &gt; <code>shareWarnAtFloor</code> = 2.5% with the occupancy-clipped tail recovery (<code>tailRecoveryMs</code>, as for skew) already ≥ <code>floorPctWarn</code> (0.5% of app runtime). Skew&#39;s <code>ratioWarn</code> = 3 and the 0.5% floor are the values the task-level tail replay was tuned against. <code>warnPct</code>/<code>critPct</code> (10%/20% speculative share) and <code>floorPctWarn</code>/<code>floorPctCrit</code> (0.5%/2% of app runtime) do not set the band; they rank the straggler-vs-speculative tiers that pick which <em>metric</em> the finding reports. The data cut and skew&#39;s gate (<code>ratioWarn</code>, <code>minTasksForP95</code>, <code>dataShareMin</code>, <code>floorPctWarn</code>) are not this detector&#39;s thresholds: it reads skew&#39;s, as resolved for the run (<code>DetectorCtx.skewThresholds</code>), so a tail that data-driven is skew&#39;s unless skew&#39;s gate does not admit it, and tuning <code>skew</code> alone never drops a tail from both. A tail skew&#39;s gate admits that data does not explain is reported here without the share test, on any stage size. A straggler share has no critical tier of its own</td><td><code>info</code></td></tr><tr><td>Speculation waste</td><td><code>speculationWastedAttempts ≥ minWasted</code> = 5 <strong>and</strong> <code>speculationWasteMs ≥ minWasteMs</code> = 60 s</td><td><code>warning</code></td></tr><tr><td>Retry waste</td><td><code>wastedAttempts ≥ minWasted</code> = 3 <strong>and</strong> <code>retryWasteMs ≥ minWasteMs</code> = 30 s (attempts superseded by a later retry of the same task)</td><td><code>warning</code></td></tr><tr><td>Tiny tasks</td><td><code>taskCount ≥ minTasks</code> = 100 <strong>and</strong> <code>taskDurationP50 ≤ maxP50</code> = 500 ms <strong>and</strong> <code>taskDurationP95 ≤ maxP95</code> = 1000 ms <strong>and</strong> the stage lasts ≥ <code>stageFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length; coalescing can&#39;t save more than the stage&#39;s own duration, so every finding on a shorter stage would grade <code>info</code>, the tasks still tiny). Checked against each stage&#39;s measured per-task overhead (task wall time minus executor run time, the estimate&#39;s own input): a stage with 100+ tasks outside the P50/P95 gate would not save 0.5% of its run by coalescing, while one inside it can, and grades above <code>info</code></td><td><code>info</code></td></tr><tr><td>Duplicate plan subtree</td><td>a subtree of ≥ <code>minSubtreeSize</code> = 3 nodes whose shape fingerprint repeats ≥ <code>minOccurrences</code> = 2× in the plan, unless its linked stages together lasted less than <code>stageFloorPct</code> = 0.5% of the run (a repeat with no linked stage time, or an unknown run duration, is kept; the claim counts at most each stage&#39;s own task-active time, so such a repeat would grade <code>info</code>, the repeat still in the plan). <code>occurrencesIdentical</code> records whether the repeats also agree node-for-node on normalized detail, ignoring AQE query-stage numbers; when they don&#39;t (same shape over another table, filter or projection) the finding is <code>info</code> with confidence <code>low</code> and no time claim. <code>stageShares</code> gives, per stage, the repeated operators&#39; share of the stage&#39;s operators (WholeStageCodegen wrappers and Exchange write halves not counted); with no attributed stage the finding is <code>info</code></td><td><code>warning</code></td></tr><tr><td>Small files read/write</td><td>per read/write side: file count &gt; <code>minFiles</code> = 100 <strong>and</strong> average file size &lt; <code>maxAvgFileSizeMB</code> = 3 MiB</td><td><code>warning</code></td></tr><tr><td>Nested loop join</td><td>a <code>BroadcastNestedLoopJoin</code> or <code>CartesianProduct</code> whose executor-side <code>number of output rows</code> ≥ <code>minOutputRows</code> = 1,000,000; a <code>BroadcastNestedLoopJoin</code> also needs that output ≥ <code>minExpansion</code> = 10 × the larger of its two inputs&#39; rows (found by walking row-preserving operators down to the nearest <code>number of output rows</code>; an input with none, or one behind an operator that can change the row count, means no finding). A <code>CartesianProduct</code> re-reads its inputs once per partition of the other side, so its input counts are not compared and the evidence carries <code>leftRows</code> and <code>rightRows</code> as null. The band comes from the wall-clock estimate over the <code>taskActiveMs</code> of the stages that run the join, with an <code>info</code> fallback</td><td><code>info</code></td></tr><tr><td>Python UDF</td><td>a SQL execution whose <code>BatchEvalPython</code> nodes (never <code>ArrowEvalPython</code>) report <code>data sent to Python workers</code> ≥ <code>minBytesSent</code> = 64 MiB <strong>and</strong> whose stages together ran ≥ <code>minStageMs</code> = 30 s. The floors are conservative guesses: no corpus or private log runs a Python UDF, so they are not tuned against real workloads. A node with no <code>data sent to Python workers</code> value is skipped</td><td><code>info</code></td></tr><tr><td>Broadcast sizing: missed</td><td>a 2-child <code>SortMergeJoin</code> with a buildable side (the right side for Inner, Cross, LeftOuter, LeftSingle, LeftSemi, LeftAnti and ExistenceJoin; the left side for Inner, Cross and RightOuter; neither for FullOuter) whose shuffle <code>data size</code> is ≥ <code>minSmallerSideBytes</code> = 1 MiB, ≤ <code>overBroadcastBytes</code> and ≤ the other side&#39;s (both sides need a shuffle <code>data size</code> of their own), and that is either at or under the effective threshold (<code>spark.sql.adaptive.autoBroadcastJoinThreshold</code> when the plan is adaptive and it is set, else <code>spark.sql.autoBroadcastJoinThreshold</code>, with the query&#39;s own settings applied) or &lt; 10 MiB, or &lt; 100 MiB with the other side &gt; 10 GiB, or &lt; 1 GiB with the other side &gt; 300 GiB (<code>broadcastTiers</code> × <code>comparisonTiers</code>)</td><td><code>info</code></td></tr><tr><td>Broadcast sizing: oversized</td><td>a <code>BroadcastExchange</code> node whose <code>data size</code> metric &gt; <code>overBroadcastBytes</code> = 1 GiB</td><td><code>warning</code></td></tr></tbody></table><h4 id="spill-magnitude-tiers" tabindex="-1">Spill magnitude tiers <a class="header-anchor" href="#spill-magnitude-tiers" aria-label="Permalink to &quot;Spill magnitude tiers&quot;">​</a></h4><p>The spill row&#39;s band is a fixed <code>warning</code> fallback, but <code>computeSpillMagnitude</code> (<code>packages/core/src/detectors.ts</code>) runs on every spill finding and sets its <code>spillMagnitude</code> field, which the Spill widget displays. Its tiers, in evaluation order (first match wins, <code>null</code> when nothing matches):</p><table tabindex="0"><thead><tr><th>Condition</th><th>Magnitude</th></tr></thead><tbody><tr><td>Single-task stage: <code>spillDiskMax ≥ singleTaskDiskGiB</code> = 1 GiB <strong>or</strong> <code>spillMemMax ≥ singleTaskMemGiB</code> = 4 GiB</td><td><code>severe</code></td></tr><tr><td>Multi-task stage: <code>spillDiskMax ≥ highDiskGiB</code> = 1 GiB, <strong>or</strong> <code>spillDiskMax / taskCount ≥ highTaskDiskMB</code> = 512 MiB (per-task proxy), <strong>or</strong> <code>spillMemMax ≥ highMemGiB</code> = 4 GiB</td><td><code>high</code></td></tr><tr><td>Multi-task stage: <code>spillDiskMax ≥ medDiskMB</code> = 256 MiB <strong>or</strong> <code>spillMemMax ≥ medMemGiB</code> = 1 GiB</td><td><code>medium</code></td></tr><tr><td>Skew (<code>taskCount ≥ skewMinTasks</code> = 10): <code>spillDiskMax / spillDiskP50 &gt; skewRatio</code> = 5× <strong>and</strong> <code>spillDiskMax ≥ skewDiskFloorMB</code> = 128 MiB</td><td><code>high</code></td></tr><tr><td>Skew (<code>taskCount ≥ 10</code>): <code>spillMemMax / spillMemP50 &gt; 5×</code> <strong>and</strong> <code>spillMemMax ≥ skewMemFloorMB</code> = 256 MiB</td><td><code>medium</code></td></tr></tbody></table><p>Disk spill is weighted worse than memory spill by design: the memory thresholds sit well above their disk counterparts at every tier.</p><h3 id="full-threshold-table-never-wallclock-derived" tabindex="-1">Full threshold table (never wallClock-derived) <a class="header-anchor" href="#full-threshold-table-never-wallclock-derived" aria-label="Permalink to &quot;Full threshold table (never wallClock-derived)&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Rule</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Stage shape: PRatio</td><td><code>taskCount / totalCores &lt; 0.5</code> (info, under-parallelized), on a stage lasting ≥ <code>lowParallelismFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length): parallelizing can&#39;t save more than the stage&#39;s duration, so a shorter stage would fire below the floor</td><td>none</td></tr><tr><td>Stage shape: OIRatio</td><td><code>outputBytes / inputBytes &gt; 10×</code> (info, data explosion)</td><td>none</td></tr><tr><td>Stage shape: TaskStageSkew</td><td><code>taskDurationMax / stageDuration &gt; stageShareMin</code> = 0.5 <strong>and</strong> <code>taskDurationMax / taskDurationP50 &gt; skewWarn</code> = 3× (info), on a multi-task stage lasting ≥ <code>taskStageSkewFloorPct</code> = 0.5% of the run. A task runs inside its stage, so the share is at most 1; the median gate is needed because on a single wave of even tasks the longest one spans the whole stage too</td><td>none</td></tr><tr><td>Failed tasks</td><td>failure rate &gt; 5% (min 10 tasks)</td><td>&gt; 20%</td></tr><tr><td>Stage failed outright</td><td>none</td><td>any <code>stageFailureReason</code> present</td></tr><tr><td>Slow host: multi-dimensional</td><td>max/median ratio across taskTime/inputBytes/shuffleBytes/storageMemory ≥ 1.33× (info); each dimension&#39;s sample must also clear an absolute floor (1000 ms for taskTime, 64 MiB for the byte dimensions). A stage lasting under <code>stageFloorPct</code> = 0.5% of the run is skipped (see the mean-duration row): a byte dimension there has no time estimate and would otherwise keep its ratio tier (a shorter stage&#39;s byte dimension would otherwise keep a warning or critical tier). The taskTime dimension also has to pass a chance test, since the slowest of many executors always tops the median: on the log scale of task duration, (the executor&#39;s mean − the median executor&#39;s mean) ÷ (the stage&#39;s task spread ÷ √ the executor&#39;s task count) must reach √(2 ln(executors ÷ (2 × <code>outlierAlpha</code>))) with <code>outlierAlpha</code> = 0.005 (3.4 for 3 executors, 4.2 for 75), so an executor with fewer tasks or a stage with more executors needs a larger gap. The stage spread is rebuilt from each executor&#39;s <code>logDurationSum</code> and <code>logDurationM2</code> (the sum and M2 of ln(1 + task duration) that <code>finalizeStage</code> accumulates); a stage with under two tasks, no spread, or an executor missing or non-finite in either field flags no executor on task time. Of the executors that pass, the one with the largest mean is reported. The taskTime dimension falls back to <code>info</code> and takes its band from its wall-clock estimate; the byte dimensions get no time estimate and keep the ratio tier</td><td>≥ <code>ratioTiers[1]</code> = 1.78× warning, ≥ <code>ratioTiers[3]</code> = 10× critical (<code>ratioTiers[2]</code> = 3.16 is not read)</td></tr><tr><td>Utilization</td><td>busy core time / allocated core time &lt; 60% (info)</td><td>none</td></tr><tr><td>Autoscaling churn: short-lived executors</td><td>&gt; 30% of executors alive under 2 min (min 5 executors)</td><td>&gt; 60%</td></tr><tr><td>Job failure rate</td><td>≥ 30% (≥ 10% info)</td><td>≥ 50%</td></tr><tr><td>Idle cores</td><td>idle share of allocated core time &gt; 50% (warning)</td><td>none</td></tr><tr><td>Memory band</td><td>highest executor peak heap / allocated &lt; 70% over-provisioned (info); no near-capacity band, since heap used counts uncollected garbage</td><td>none</td></tr><tr><td>Caching opportunity</td><td>same input relation (or join/union subtree) scanned by ≥ <code>minExecutions</code> = 2 SQL executions in one run</td><td>none (single tier, info)</td></tr><tr><td>Cache utilization: partial caching</td><td><code>numCachedPartitions / numPartitions &lt; 0.90</code> (info)</td><td><code>&lt; 0.50</code> (warning)</td></tr><tr><td>Cache utilization: disk spillover</td><td><code>diskSize / (memorySize + diskSize) &gt; 0.15</code> (info), <code>MEMORY_AND_DISK*</code> only</td><td><code>&gt; 0.40</code> (warning)</td></tr><tr><td>Cache utilization: storage unobserved</td><td>persisted RDDs, but no <code>SparkListenerBlockUpdated</code> for any <code>rdd_*</code> block and every RDD Info figure 0 (<code>spark.eventLog.logBlockUpdates.enabled</code> off on Spark 2.3+): a missing-evidence caveat, not a threshold</td><td>none (single tier, info)</td></tr><tr><td>Core locality</td><td>non-local task ratio ≥ <code>warnRatio</code> = 15% (min <code>minTasks</code> = 50 tasks)</td><td>≥ <code>critRatio</code> = 35%</td></tr><tr><td>Config: memory overhead</td><td><code>spark.executor.memoryOverhead</code> below max(minimum, factor × executor memory) (info). The minimum is <code>spark.executor.minMemoryOverhead</code> on Spark 4+, else <code>floorMB</code> = 384 MiB; the factor is <code>spark.executor.memoryOverheadFactor</code> on Spark 3.3+, else <code>floorPct</code> = 10%. An unrecorded Spark version reads both settings</td><td>none</td></tr></tbody></table><p>The RDD Info cache figures on stage events (<code>Number of Cached Partitions</code>, <code>Memory Size</code>, <code>Disk Size</code>) are always 0 since Spark 2.3; Spark 1.x fills them only on <code>StageCompleted</code>. Real cache evidence is <code>SparkListenerBlockUpdated</code>, written only with <code>spark.eventLog.logBlockUpdates.enabled=true</code> (<code>recordBlockUpdate</code> in <code>event-handlers.ts</code>); the corpus <code>cache-memory-only</code>/<code>cache-memory-and-disk</code> logs carry it. Count a block&#39;s sizes only where its storage level says it lives: a drop to disk still reports the dropped bytes as <code>Memory Size</code>.</p><p>Spill classification: ≥80% tasks with zero spill → <code>skew</code>; &lt;20% zero → <code>volume</code>; else <code>unclassified</code>. The classification badge is always shown in both compact and expanded spill widget states. This is independent of the magnitude tiers above: classification says <em>what kind</em> of spill, magnitude says <em>how much</em>.</p><h4 id="evidence-fields-stagefailed-retrywaste" tabindex="-1">Evidence fields (<code>stageFailed</code> / <code>retryWaste</code>) <a class="header-anchor" href="#evidence-fields-stagefailed-retrywaste" aria-label="Permalink to &quot;Evidence fields (`stageFailed` / `retryWaste`)&quot;">​</a></h4><p>Beyond a scalar <code>metric</code>/<code>value</code>, both entries&#39; <code>detect()</code> attach to the finding:</p><ul><li><code>numTasks</code>: <code>stage.taskCount</code> at detection time.</li><li><code>memoryBytesSpilled</code>: <code>stage.memoryBytesSpilled</code> at detection time.</li><li><code>stageFailed</code> only: <code>failedTaskDetails</code>: up to 20 <code>FailedTaskSample</code> records (<code>taskId</code>, <code>attemptNumber</code>, <code>host</code>, <code>executorId</code>, <code>reason</code>, <code>peakExecMem</code>, <code>memSpilled</code>, <code>shuffleWrite</code>) for tasks still marked failed when the stage was finalized (<code>finalizeStage</code>, <code>stage-quantiles.ts</code>).</li><li><code>retryWaste</code> only: <code>retriedTaskDetails</code>: up to 20 <code>FailedTaskSample</code> records for attempts discarded by the retry-dedup logic in <code>accumulateTask</code> (<code>event-handlers.ts</code>): captured at the moment they&#39;d otherwise be thrown away, since by finalize time only the winning attempt survives.</li></ul><p>Both sample arrays are capped at 20 entries, filled in first-encountered order (finalize order for <code>failedTaskDetails</code>, discard order for <code>retriedTaskDetails</code>), not spread across distinct hosts/executors: a stage with failures clustered on one bad host could fill the cap before a more informative failure elsewhere in the stage is ever sampled.</p><h4 id="evidence-fields-failures" tabindex="-1">Evidence fields (<code>failures</code>) <a class="header-anchor" href="#evidence-fields-failures" aria-label="Permalink to &quot;Evidence fields (`failures`)&quot;">​</a></h4><ul><li><code>dominantReason</code>: the most frequent end-reason tag (<code>ExceptionFailure</code>, <code>ExecutorLostFailure</code>, ...) among tasks still failed at finalize.</li><li><code>dominantError</code>: the error behind that tag, from its largest failure group: the exception class, or <code>&lt;tag&gt;: &lt;loss reason&gt;</code>; falls back to the tag. The recommendation names it and never embeds a message.</li><li><code>failureGroups</code>: up to 5 <code>TaskFailureGroup</code>s (<code>reason</code>, <code>className</code>, <code>message</code>, <code>lossReason</code>, <code>stackExcerpt</code>, <code>count</code>), most frequent first: one group per distinct tag, class, message and loss reason, one excerpt each.</li><li><code>otherFailedTasks</code>: failed tasks that no shown group covers.</li></ul><p><code>packages/core/src/task-failure.ts</code> reads the details from <code>Task End Reason</code> (<code>Class Name</code>, <code>Description</code>, <code>Full Stack Trace</code>, <code>Loss Reason</code>, FetchFailed <code>Message</code>, <code>Kill Reason</code>) and bounds them at ingest: a message or loss reason to its first line (for a Python traceback, its final error line), 300 characters; an excerpt to the header, 8 frames, a Python traceback&#39;s error line and the last <code>Caused by:</code> line, 2000 characters in all. While parsing, a stage keeps at most 50 distinct failures (<code>StageRecord.failureDetails</code>, freed at finalize); later ones count only toward the tag. Redaction (<code>redact.ts</code>) replaces every <code>failureGroups[].message</code> and strips message text from <code>stackExcerpt</code>.</p><h3 id="threshold-sources" tabindex="-1">Threshold sources <a class="header-anchor" href="#threshold-sources" aria-label="Permalink to &quot;Threshold sources&quot;">​</a></h3><p>Where each threshold that is not read from Spark&#39;s own source comes from.</p><table tabindex="0"><thead><tr><th>Threshold</th><th>Source status</th></tr></thead><tbody><tr><td><code>IMPACT_FLOOR_PCT_WARN</code> = 0.5% and <code>IMPACT_FLOOR_PCT_CRIT</code> = 2% (<code>impact-band.ts</code>), and every runtime floor that defaults to them or to 0.5% (<code>floorPctWarn</code>, <code>floorPctCrit</code>, <code>stageFloorPct</code>, <code>lowInfoFloorPct</code>, <code>lowParallelismFloorPct</code>, <code>taskStageSkewFloorPct</code>)</td><td>Our own noise floors, unvalidated</td></tr><tr><td><code>skew</code> <code>dataShareMin</code> = 0.5</td><td>Our own majority cut</td></tr><tr><td><code>gc</code> <code>minRunTimeMs</code> = 10 s</td><td>Our own noise floor, so a stage that barely ran flags in neither direction</td></tr><tr><td><code>gc</code> <code>warnPct100</code> = 10 and <code>lowInfoPct100</code> = 5</td><td>Carried over unchanged from an external GC heuristic</td></tr><tr><td><code>memoryUtilization</code> <code>idleCoreWarn</code> = 0.50</td><td>Carried over from an external wasted-cores heuristic</td></tr><tr><td><code>memoryUtilization</code> <code>wasteBufferMultiplier</code> = 1.5</td><td>Unverified buffer</td></tr><tr><td><code>nestedLoopJoin</code> <code>minOutputRows</code> = 1,000,000 and <code>minExpansion</code> = 10</td><td>Our own noise floors; the stage time the finding is graded on decides how much it matters</td></tr><tr><td><code>pythonUdf</code> <code>minBytesSent</code> and <code>minStageMs</code></td><td>Conservative guesses, not tuned against a real workload</td></tr><tr><td>The assumed throughputs and overheads in <code>impact-model.ts</code></td><td>Starting assumptions, unvalidated</td></tr></tbody></table><h3 id="spark-defaults-by-release" tabindex="-1">Spark defaults by release <a class="header-anchor" href="#spark-defaults-by-release" aria-label="Permalink to &quot;Spark defaults by release&quot;">​</a></h3><p><code>sparkConfDefault()</code> (<code>packages/core/src/spark-conf.ts</code>) answers a property&#39;s default for the run&#39;s Spark version from this table. A property introduced by a release has no default before it.</p><table tabindex="0"><thead><tr><th>Property</th><th>Default</th><th>From</th></tr></thead><tbody><tr><td><code>spark.speculation.multiplier</code></td><td>1.5, then 3</td><td>4.0.0</td></tr><tr><td><code>spark.speculation.quantile</code></td><td>0.75, then 0.9</td><td>4.0.0</td></tr><tr><td><code>spark.sql.adaptive.enabled</code></td><td>false, then true</td><td>3.2.0</td></tr><tr><td><code>spark.sql.adaptive.skewJoin.enabled</code>, <code>spark.sql.adaptive.coalescePartitions.enabled</code></td><td>true</td><td>3.0.0</td></tr><tr><td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> = 5, <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> = 256MB</td><td>A partition is skewed above both the factor times the median partition and the threshold</td><td>3.0.0</td></tr><tr><td><code>spark.sql.adaptive.forceOptimizeSkewedJoin</code></td><td>false; true accepts the extra shuffle a split can cost</td><td>3.3.0</td></tr><tr><td><code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code></td><td>64MB (falls back to <code>spark.sql.adaptive.shuffle.targetPostShuffleInputSize</code>)</td><td>3.0.0</td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.parallelismFirst</code></td><td>true; AQE then sizes merged tasks from <code>defaultParallelism</code> too, so a larger advisory size alone does not merge more</td><td>3.2.0</td></tr><tr><td><code>spark.executor.memoryOverheadFactor</code></td><td>0.1 (0.4 for Kubernetes non-JVM jobs, which the log does not tell apart); overhead is max(factor x executor memory, minimum) when <code>spark.executor.memoryOverhead</code> is unset</td><td>3.3.0</td></tr><tr><td><code>spark.executor.minMemoryOverhead</code></td><td>384m</td><td>4.0.0</td></tr><tr><td><code>spark.sql.execution.pythonUDF.arrow.enabled</code></td><td>false, then true</td><td>3.4.0 (introduced), 4.2.0 (on)</td></tr></tbody></table><p>The remaining keys in the table (<code>spark.speculation</code>, <code>spark.executor.memory</code>, <code>spark.serializer</code>, <code>spark.shuffle.service.enabled</code>, the <code>spark.eventLog</code> switches, the dynamic-allocation bounds, <code>spark.sql.shuffle.partitions</code>, <code>spark.sql.autoBroadcastJoinThreshold</code>, <code>spark.sql.files.maxPartitionBytes</code>) have one default on every release.</p><h3 id="calibration-data" tabindex="-1">Calibration data <a class="header-anchor" href="#calibration-data" aria-label="Permalink to &quot;Calibration data&quot;">​</a></h3><p>Calibrated on 14 real logs. Each figure counts findings or groups the default thresholds produce on them. A 0.5% runtime floor drops findings on stages that barely ran, where the claim is clipped to the stage and would grade <code>info</code>; the condition is still true there.</p><table tabindex="0"><thead><tr><th>Detector</th><th>Calibration figure</th></tr></thead><tbody><tr><td><code>duplicatePlanSubtree</code></td><td>270 of 546 duplicate groups differ beyond AQE query-stage numbers (<code>occurrencesIdentical: false</code>); 340 of 545 findings sit on repeats whose stages are shorter than the floor; a WholeStageCodegen wrapper or an Exchange write half is the only operator outside the subtree on 426 stages</td></tr><tr><td><code>stageShape</code> (<code>lowParallelism</code>)</td><td>2,839 of 3,005 findings fall under the floor, 2,168 of them on sub-second stages</td></tr><tr><td><code>shuffle</code></td><td>182 of 284 findings fall under the floor</td></tr><tr><td><code>spill</code></td><td>12 of 38 findings fall under the floor, all <code>info</code></td></tr><tr><td><code>gc</code> (low GC)</td><td>464 of 685 low-GC notes fall under the floor (701 <code>gc</code> findings in all)</td></tr><tr><td><code>slowHost</code></td><td>324 of 452 findings fall under the floor, all <code>info</code></td></tr><tr><td><code>straggler</code></td><td>671 of 753 findings fall under the floor, none above <code>info</code>. Scored against a task-level replay of every stage (recoverable = replay with each task over 4x P50 capped at P50), admitting 2.5-5% shares whose clipped tail already clears <code>floorPctWarn</code> finds 3 stages (10-48 s) for 1 borderline miss; admitting every 2.5% share adds 86 findings below the floor</td></tr><tr><td><code>tinyTask</code></td><td>132 of 151 findings fall under the floor</td></tr><tr><td><code>coldStart</code></td><td>The driver&#39;s own startup before the first job takes 36-47 s on every log, so the gap is measured from the first stage&#39;s submission, not from application start</td></tr></tbody></table>',33)])])}const f=t(s,[["render",r]]);export{g as __pageData,f as default};

package/public/docs/assets/{contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.lean.js → contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.lean.js}

@@ -1 +1 @@
-import{_ as t,o,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Threshold tables","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract/threshold-tables.md","filePath":"contributor-guide/architecture/detector-contract/threshold-tables.md"}'),s={name:"contributor-guide/architecture/detector-contract/threshold-tables.md"};function r(i,e,c,n,l,h){return o(),d("div",null,[...e[0]||(e[0]=[a("",23)])])}const m=t(s,[["render",r]]);export{u as __pageData,m as default};
+import{_ as t,o,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Threshold tables","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/detector-contract/threshold-tables.md","filePath":"contributor-guide/architecture/detector-contract/threshold-tables.md"}'),s={name:"contributor-guide/architecture/detector-contract/threshold-tables.md"};function r(i,e,c,n,l,h){return o(),d("div",null,[...e[0]||(e[0]=[a("",33)])])}const f=t(s,[["render",r]]);export{g as __pageData,f as default};

package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.js}

@@ -1 +1 @@
-import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Drill-down","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/drill-down.md","filePath":"contributor-guide/architecture/drill-down.md"}'),c={name:"contributor-guide/architecture/drill-down.md"};function d(n,e,i,r,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s('<h1 id="drill-down" tabindex="-1">Drill-down <a class="header-anchor" href="#drill-down" aria-label="Permalink to &quot;Drill-down&quot;">​</a></h1><p><code>StageDetailProvider</code> (<code>src/view/StageDetailContext.tsx</code>) exposes stage drill-down through React context. Any component calls <code>useStageDetail().openStage(stageId)</code> to open <code>StageDetailDialog.tsx</code>, a shadcn <code>Dialog</code> (base-ui), for that stage. The direct callers are StagePill, Timeline, StageTable and RunVerdict (a step&#39;s stage button). Every widget that embeds a <code>StagePill</code>, <code>StagePillGroup</code> or <code>StageHeader</code> reaches it indirectly.</p><p>The dialog is titled &quot;Stage N&quot;, with Spark&#39;s stage name (the code line that created the stage) as a labelled &quot;Code location&quot; description. Its body opens with one sentence placing the stage in the run (duration, share of the run&#39;s wall-clock, task count, how many finding types it carries), then lists each finding type at the stage (<code>locationKey</code>&#39;s rule, the one the verdict groups steps by: a <code>stageId</code> match, or a <code>stageIds</code> list naming only this stage) the way <code>RunVerdict</code> lists a step: tag and action label, the measurement (&quot;What&#39;s happening&quot;, split off by <code>recommendationParts</code>), &quot;What to try&quot;, the impact estimate, and a <strong>Show evidence</strong> button. Types follow the verdict&#39;s own order (<code>buildNextSteps</code>: potential savings first, failure findings first on a run whose jobs failed), with types outside the stage&#39;s own verdict step after them, worst band first. The grouping and order are computed once by <code>interpretRun</code> (<code>interpretStages</code> in <code>packages/core/src/run-interpretation.ts</code>, shipped as <code>stages[id].{findingIndexes,typeOrder}</code>); the dialog only renders them. Show evidence closes the dialog and calls the optional <code>onRoute</code> prop (<code>routeToVisible</code> from <code>Dashboard.tsx</code>, which clears any board filter that hides the target, as the verdict&#39;s own route does); <code>finalFocus</code> skips returning focus to the opener in that case, so the route&#39;s own focus on the evidence stands. Without <code>onRoute</code> the button is not shown.</p><h2 id="plan-dot-serialization" tabindex="-1">Plan DOT serialization <a class="header-anchor" href="#plan-dot-serialization" aria-label="Permalink to &quot;Plan DOT serialization&quot;">​</a></h2><p><code>packages/core/src/plan-dot.ts</code> (entry <code>planTreeToDot(planTree, { title })</code>) serializes a resolved <code>planTree</code> to a Graphviz DOT string. It is dependency-free string building. One pre-order walk (<code>walkPlanTree</code>, deduping shared subtrees) assigns each node a stable id and <code>label</code> (name, plus <code>detail</code> on a second line when it differs) and records its parent→child edge, since the parent&#39;s id always exists first. Edges are written after all node lines. The graph is laid out <code>rankdir=BT</code>, leaves at the bottom, matching Spark&#39;s own plan orientation.</p><p>It carries no metric annotation: pure structure. A null plan returns an empty string. There is no download/export UI for this output; <code>PlanView.tsx</code> calls it only to decide whether a stage&#39;s plan tree can render as a graph at all, and a non-empty result gates the &quot;View plan graph&quot; button.</p><h2 id="disclosure-hierarchy" tabindex="-1">Disclosure hierarchy <a class="header-anchor" href="#disclosure-hierarchy" aria-label="Permalink to &quot;Disclosure hierarchy&quot;">​</a></h2><p>The Summary/Context/Details 3-tier framing (collapsed lead metric → expanded widget body → per-stage <code>StageDetailDialog</code>) does not apply uniformly across the 28 registry widgets (<code>src/view/detector-registry.tsx</code>). 18 have a stage-anchored Details tier reachable via <code>StagePill</code>/<code>StagePillGroup</code>: Skew, StageShape, TinyTask, ShuffleIO, PartitionSizing, Spill, GcPressure, StageFailed, TaskFailures, RetryWaste, SlowHost, StageSlowness, Straggler, SpeculationWaste, and DuplicatePlanSubtree, SmallFiles, UnderBroadcast and OverBroadcast (sql-scope, so they list every stage in <code>stageIds</code> as a <code>StagePillGroup</code> rather than one <code>StagePill</code>). The other 10 are app- or config-scope with no stage to drill into, by design, so they stop at Summary/Context: MemoryUtilization, ExecutorUtilization (<code>utilization</code> is an app-wide average, no stage), JobFailures, ConfigAudit, CacheUtilization, CoreUsageArea, AutoscalingChurn, CachingOpportunity (<code>scope: &#39;app&#39;</code>, <code>stageId: null</code> on both its finding constructions, so it has no stage to anchor to despite reading like a per-stage widget), IncompleteRun, and ColdStart (unlike SlowHost, StageSlowness, Straggler and SpeculationWaste, app-scoped with no <code>stageId</code>).</p><h2 id="reference-panel" tabindex="-1">Reference panel <a class="header-anchor" href="#reference-panel" aria-label="Permalink to &quot;Reference panel&quot;">​</a></h2><p><code>DocsLink</code> (<code>src/view/DocsContext.tsx</code>), finding tag pills (<code>TagBadge</code>) and a few guide links call <code>useDocs().open(anchor)</code> or <code>openSite(path)</code> to open a docked, resizable docs panel (<code>src/view/DocsSheet.tsx</code>, mounted once in <code>src/App.tsx</code> inside <code>DocsProvider</code>). The topbar&#39;s Docs button is a plain link that opens the docs site in a new tab. The panel iframes a docs-site (VitePress) page, built from the tuning reference under <code>packages/core/src/docs-content/{chapters,tuning,diagrams}</code> (generated at build and test time by <code>scripts/fetch-tuning-docs.mjs</code>, gitignored, from the <code>shuffle-works/spark-tuning-reference</code> commit pinned in the committed <code>docs-content/upstream.json</code>; <code>npm run docs:bump</code> moves the pin) and published as static HTML at <code>docs/tuning-reference/&lt;page&gt;.html</code> (<code>docs-config.ts</code>&#39;s <code>docsUrl()</code> resolves an anchor to that path plus a <code>#&lt;anchor&gt;</code> fragment). <code>open</code>/<code>openSite</code> set React state (<code>isOpen</code>, <code>target</code>). The panel is a non-modal base-ui <code>Dialog</code> (<code>modal={false}</code>, <code>disablePointerDismissal</code>) laid out with <code>react-resizable-panels</code>. The dashboard stays interactive and reflows beside it through the <code>--docs-inset</code> CSS variable. Only the close button or Escape dismisses it, and base-ui owns focus-in, focus-restore and Escape. Key events inside the iframe never reach the app&#39;s document, so <code>DocsSheet</code>&#39;s <code>listenForEscapeInFrame</code> also listens in a same-origin frame&#39;s own document and closes on Escape unless the docs&#39; search popup is open (a cross-origin frame keeps only the close button). Exported dashboards have no docs, so they never open this panel. There is a single <code>DocsTarget</code> shape (<code>{ kind: &#39;site&#39;, source: &#39;reference&#39; | &#39;guide&#39;, path }</code>, set by <code>open(anchor)</code> or <code>openSite(path)</code>; <code>source</code> only picks the panel title), so <code>DocsSheet</code> always drives the iframe the same way, reassigning <code>src</code> on any path or theme change.</p><p>Most anchors the app links to are the page of the same name; a handful are in-page fragments on another page instead (config-audit sub-findings and the metric glossary live on the <code>config</code>/<code>metrics</code> pages; bottleneck sub-anchors such as <code>bottleneck-stage-shape</code> live on the page of the bottleneck that owns them, and two, <code>bottleneck-autoscaling-churn</code> and <code>bottleneck-cache-utilization</code>, on the <code>cluster-config</code>/<code>memory-model</code> chapters): <code>docs-config.ts</code>&#39;s <code>pageForAnchor()</code> is the one place that resolves an anchor to its owning page. <code>npm run docs:build</code> (run automatically by <code>npm run build</code>) generates <code>packages/core/src/docs-content/</code> from the pin if needed (<code>scripts/fetch-tuning-docs.mjs</code>), reshapes it into <code>docs-site/tuning-reference/*.md</code> (<code>scripts/build-tuning-reference.mjs</code>), then VitePress renders it into <code>docs-site/.vitepress/dist</code>, and the <code>copyDocsSite</code> plugin (<code>vite-plugins/copy-docs-site.ts</code>, registered in <code>vite.config.ts</code>) copies that output to <code>dist/docs</code>; Vite&#39;s relative asset base keeps the app and docs usable when <code>dist/</code> is deployed under a URL subpath. <code>tests/doc-anchor-coverage.test.js</code> intersects <code>packages/core/src/docs-content/chapters/nav-index.json</code> against every detector&#39;s <code>docAnchor</code> and warns (never fails) on dead links (detector points at an anchor the nav index doesn&#39;t have) or orphaned Detector Catalog anchors (no detector points at them); see <code>scripts/doc-anchor-coverage.js</code>. The landing page (<code>docs-site/tuning-reference/index.md</code>, the symptom-picker entry page) is hand-authored, committed markdown, unlike the generated pages next to it.</p><p>A docs-site page has no channel back to this app: it&#39;s a plain static page with no <code>postMessage</code> listener. <code>DocsSheet.tsx</code> reassigns the iframe&#39;s <code>src</code> outright on any change to the resolved path or the theme, forcing a full reload. Since re-assigning the exact same <code>src</code> string wouldn&#39;t make the browser reload it, a <code>t=&lt;theme&gt;</code> marker is threaded into the query string ahead of the <code>#anchor</code> hash purely to change the string and force a real reload; the page never reads that param itself: it reads its light/dark preference once, from the <code>vitepress-theme-appearance</code> localStorage key <code>ThemeProvider</code> keeps current, the moment it boots.</p>',12)])])}const u=o(c,[["render",d]]);export{g as __pageData,u as default};
+import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Drill-down","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/drill-down.md","filePath":"contributor-guide/architecture/drill-down.md"}'),c={name:"contributor-guide/architecture/drill-down.md"};function d(n,e,i,r,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s('<h1 id="drill-down" tabindex="-1">Drill-down <a class="header-anchor" href="#drill-down" aria-label="Permalink to &quot;Drill-down&quot;">​</a></h1><p><code>StageDetailProvider</code> (<code>src/view/StageDetailContext.tsx</code>) exposes stage drill-down through React context. Any component calls <code>useStageDetail().openStage(stageId)</code> to open <code>StageDetailDialog.tsx</code>, a shadcn <code>Dialog</code> (base-ui), for that stage. The direct callers are StagePill, Timeline, StageTable and RunVerdict (a step&#39;s stage button). Every widget that embeds a <code>StagePill</code>, <code>StagePillGroup</code> or <code>StageHeader</code> reaches it indirectly.</p><p>The dialog is titled &quot;Stage N&quot;, with Spark&#39;s stage name (the code line that created the stage) as a labelled &quot;Code location&quot; description. Its body opens with one sentence placing the stage in the run (duration, share of the run&#39;s wall-clock, task count, how many finding types it carries), then lists each finding type at the stage (<code>locationKey</code>&#39;s rule, the one the verdict groups steps by: a <code>stageId</code> match, or a <code>stageIds</code> list naming only this stage) the way <code>RunVerdict</code> lists a step: tag and action label, the measurement (&quot;What&#39;s happening&quot;, split off by <code>recommendationParts</code>), &quot;What to try&quot;, the impact estimate, and a <strong>Show evidence</strong> button. Types follow the verdict&#39;s own order (<code>buildNextSteps</code>: potential savings first, failure findings first on a run whose jobs failed), with types outside the stage&#39;s own verdict step after them, worst band first. The grouping and order are computed once by <code>interpretRun</code> (<code>interpretStages</code> in <code>packages/core/src/run-interpretation.ts</code>, shipped as <code>stages[id].{findingIndexes,typeOrder}</code>); the dialog only renders them. Show evidence closes the dialog and calls the optional <code>onRoute</code> prop (<code>routeToVisible</code> from <code>Dashboard.tsx</code>, which clears any board filter that hides the target, as the verdict&#39;s own route does); <code>finalFocus</code> skips returning focus to the opener in that case, so the route&#39;s own focus on the evidence stands. Without <code>onRoute</code> the button is not shown.</p><h2 id="plan-dot-serialization" tabindex="-1">Plan DOT serialization <a class="header-anchor" href="#plan-dot-serialization" aria-label="Permalink to &quot;Plan DOT serialization&quot;">​</a></h2><p><code>packages/core/src/plan-dot.ts</code> (entry <code>planTreeToDot(planTree, { title })</code>) serializes a resolved <code>planTree</code> to a Graphviz DOT string. It is dependency-free string building. One pre-order walk (<code>walkPlanTree</code>, deduping shared subtrees) assigns each node a stable id and <code>label</code> (name, plus <code>detail</code> on a second line when it differs) and records its parent→child edge, since the parent&#39;s id always exists first. Edges are written after all node lines. The graph is laid out <code>rankdir=BT</code>, leaves at the bottom, matching Spark&#39;s own plan orientation.</p><p>It carries no metric annotation: pure structure. A null plan returns an empty string. There is no download/export UI for this output; <code>PlanView.tsx</code> calls it only to decide whether a stage&#39;s plan tree can render as a graph at all, and a non-empty result gates the &quot;View plan graph&quot; button.</p><h2 id="disclosure-hierarchy" tabindex="-1">Disclosure hierarchy <a class="header-anchor" href="#disclosure-hierarchy" aria-label="Permalink to &quot;Disclosure hierarchy&quot;">​</a></h2><p>The Summary/Context/Details 3-tier framing (collapsed lead metric → expanded widget body → per-stage <code>StageDetailDialog</code>) does not apply uniformly across the 28 registry widgets (<code>src/view/detector-registry.tsx</code>). 18 have a stage-anchored Details tier reachable via <code>StagePill</code>/<code>StagePillGroup</code>: Skew, StageShape, TinyTask, ShuffleIO, PartitionSizing, Spill, GcPressure, StageFailed, TaskFailures, RetryWaste, SlowHost, StageSlowness, Straggler, SpeculationWaste, and DuplicatePlanSubtree, SmallFiles, UnderBroadcast and OverBroadcast (sql-scope, so they list every stage in <code>stageIds</code> as a <code>StagePillGroup</code> rather than one <code>StagePill</code>). The other 10 are app- or config-scope with no stage to drill into, by design, so they stop at Summary/Context: MemoryUtilization, ExecutorUtilization (<code>utilization</code> is an app-wide average, no stage), JobFailures, ConfigAudit, CacheUtilization, CoreUsageArea, AutoscalingChurn, CachingOpportunity (<code>scope: &#39;app&#39;</code>, <code>stageId: null</code> on both its finding constructions, so it has no stage to anchor to despite reading like a per-stage widget), IncompleteRun, and ColdStart (unlike SlowHost, StageSlowness, Straggler and SpeculationWaste, app-scoped with no <code>stageId</code>).</p><h2 id="reference-panel" tabindex="-1">Reference panel <a class="header-anchor" href="#reference-panel" aria-label="Permalink to &quot;Reference panel&quot;">​</a></h2><p><code>DocsLink</code> (<code>src/view/DocsContext.tsx</code>), finding tag pills (<code>TagBadge</code>) and a few guide links call <code>useDocs().open(anchor)</code> or <code>openSite(path)</code> to open a docked, resizable docs panel (<code>src/view/DocsSheet.tsx</code>, mounted once in <code>src/App.tsx</code> inside <code>DocsProvider</code>). The topbar&#39;s Docs button is a plain link that opens the docs site in a new tab. The panel iframes a docs-site (VitePress) page, built from the tuning reference under <code>packages/core/src/docs-content/{chapters,tuning,diagrams}</code> (generated at build and test time by <code>scripts/fetch-tuning-docs.mjs</code>, gitignored, from the <code>shuffle-works/spark-tuning-reference</code> commit pinned in the committed <code>docs-content/upstream.json</code>; <code>npm run docs:bump</code> moves the pin) and published as static HTML at <code>docs/tuning-reference/&lt;page&gt;.html</code> (<code>docs-config.ts</code>&#39;s <code>docsUrl()</code> resolves an anchor to that path plus a <code>#&lt;anchor&gt;</code> fragment). <code>open</code>/<code>openSite</code> set React state (<code>isOpen</code>, <code>target</code>). The panel is a non-modal base-ui <code>Dialog</code> (<code>modal={false}</code>, <code>disablePointerDismissal</code>) laid out with <code>react-resizable-panels</code>. The dashboard stays interactive and reflows beside it through the <code>--docs-inset</code> CSS variable. Only the close button or Escape dismisses it, and base-ui owns focus-in, focus-restore and Escape. Key events inside the iframe never reach the app&#39;s document, so <code>DocsSheet</code>&#39;s <code>listenForEscapeInFrame</code> also listens in a same-origin frame&#39;s own document and closes on Escape unless the docs&#39; search popup is open (a cross-origin frame keeps only the close button). Exported dashboards have no docs, so they never open this panel. There is a single <code>DocsTarget</code> shape (<code>{ kind: &#39;site&#39;, source: &#39;reference&#39; | &#39;guide&#39;, path }</code>, set by <code>open(anchor)</code> or <code>openSite(path)</code>; <code>source</code> only picks the panel title), so <code>DocsSheet</code> always drives the iframe the same way, reassigning <code>src</code> on any path or theme change.</p><p>Most anchors the app links to are the page of the same name; a handful are in-page fragments on another page instead (config-audit sub-findings and the metric glossary live on the <code>config</code>/<code>metrics</code> pages; bottleneck sub-anchors such as <code>bottleneck-stage-shape</code> live on the page of the bottleneck that owns them, and two, <code>bottleneck-autoscaling-churn</code> and <code>bottleneck-cache-utilization</code>, on the <code>cluster-config</code>/<code>memory-model</code> chapters): <code>docs-config.ts</code>&#39;s <code>pageForAnchor()</code> is the one place that resolves an anchor to its owning page. <code>npm run docs:build</code> (run automatically by <code>npm run build</code>) generates <code>packages/core/src/docs-content/</code> from the pin if needed (<code>scripts/fetch-tuning-docs.mjs</code>), reshapes it into <code>docs-site/tuning-reference/*.md</code> (<code>scripts/build-tuning-reference.mjs</code>), then VitePress renders it into <code>docs-site/.vitepress/dist</code>, and the <code>copyDocsSite</code> plugin (<code>vite-plugins/copy-docs-site.ts</code>, registered in <code>vite.config.ts</code>) copies that output to <code>dist/docs</code>; Vite&#39;s relative asset base keeps the app and docs usable when <code>dist/</code> is deployed under a URL subpath. <code>tests/doc-anchor-coverage.test.js</code> intersects <code>packages/core/src/docs-content/chapters/nav-index.json</code> against every detector&#39;s <code>docAnchor</code> and warns (never fails) on dead links (detector points at an anchor the nav index doesn&#39;t have) or orphaned Detector Catalog anchors (no detector points at them); see <code>scripts/doc-anchor-coverage.js</code>. The landing page (<code>docs-site/tuning-reference/index.md</code>, the symptom-picker entry page) is hand-authored, committed markdown, unlike the generated pages next to it.</p><p>A docs-site page has no channel back to this app: it&#39;s a plain static page with no <code>postMessage</code> listener. <code>DocsSheet.tsx</code> reassigns the iframe&#39;s <code>src</code> outright on any change to the resolved path or the theme, forcing a full reload. Since re-assigning the exact same <code>src</code> string wouldn&#39;t make the browser reload it, a <code>t=&lt;theme&gt;</code> marker is threaded into the query string ahead of the <code>#anchor</code> hash purely to change the string and force a real reload; the page never reads that param itself: it reads its light/dark preference once, from the <code>vitepress-theme-appearance</code> localStorage key <code>ThemeProvider</code> keeps current, the moment it boots.</p><p>The panel&#39;s popup is <code>z-[60]</code>, above the app&#39;s other modal dialogs (<code>z-50</code>), so a dialog&#39;s backdrop blur dims the dashboard behind it without also blurring the docs.</p>',13)])])}const u=o(c,[["render",d]]);export{g as __pageData,u as default};

package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.lean.js}

@@ -1 +1 @@
-import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Drill-down","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/drill-down.md","filePath":"contributor-guide/architecture/drill-down.md"}'),c={name:"contributor-guide/architecture/drill-down.md"};function d(n,e,i,r,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s("",12)])])}const u=o(c,[["render",d]]);export{g as __pageData,u as default};
+import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Drill-down","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/drill-down.md","filePath":"contributor-guide/architecture/drill-down.md"}'),c={name:"contributor-guide/architecture/drill-down.md"};function d(n,e,i,r,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s("",13)])])}const u=o(c,[["render",d]]);export{g as __pageData,u as default};

package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.js

@@ -0,0 +1 @@
+import{_ as o,o as t,c as a,a5 as d}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Plan graph view","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/drill-down/plan-graph.md","filePath":"contributor-guide/architecture/drill-down/plan-graph.md"}'),n={name:"contributor-guide/architecture/drill-down/plan-graph.md"};function s(r,e,i,c,l,h){return t(),a("div",null,[...e[0]||(e[0]=[d('<h1 id="plan-graph-view" tabindex="-1">Plan graph view <a class="header-anchor" href="#plan-graph-view" aria-label="Permalink to &quot;Plan graph view&quot;">​</a></h1><p>How the plan graph is modelled and rendered in the drill-down.</p><p><code>buildPlanGraphModel(planTree, opts)</code> (<code>packages/core/src/plan-graph-model.ts</code>) flattens a resolved <code>planTree</code> into a <code>{ nodes, edges, segmentIndex, segmentCount, scope, segmentStageIds }</code> graph shape. <code>src/view/PlanGraphRoute.tsx</code> owns the view state and <code>src/view/plan-graph/PlanGraphCanvas.tsx</code> renders the model with <code>@xyflow/react</code> (React Flow, pan/zoom/viewport/MiniMap) and <code>@dagrejs/dagre</code> (node layout, <code>src/view/plan-graph/dagre-layout.ts</code>, <code>rankdir: &#39;RL&#39;</code>).</p><p>Every <code>PlanGraphEdge</code> points <code>source: parentId, target: id</code> (parent/consumer → child/producer), and dagre places an edge&#39;s source at the higher-rank end. So <code>RL</code>, rather than the more intuitive-looking <code>LR</code>, is what lands reads/scans (targets, computed first) on the left and the final write/root (source) on the right, matching the left-to-right reading order of the plan&#39;s data flow. Each node&#39;s React Flow <code>Handle</code>s follow the same horizontal routing: <code>type=&quot;target&quot;</code> on <code>Position.Right</code> (its parent sits to the right) and <code>type=&quot;source&quot;</code> on <code>Position.Left</code> (its children sit to the left).</p><p>The desktop layout is always this one right-to-left flow, at every canvas size: an ancestor is always to the right of its descendant, in reading order, and a long chain scrolls sideways at the readable zoom rather than wrapping into rows. Only a phone-width viewport (below Tailwind&#39;s <code>sm</code> breakpoint, 640px, <code>useNarrowViewport.ts</code>) lays a plan of at most 150 operators out <code>BT</code>, the same arrangement stacked vertically with reads/scans on top, so it scrolls down instead. A <code>BT</code> node&#39;s handles move to <code>Position.Bottom</code> (target) and <code>Position.Top</code> (source) through the <code>sourcePosition</code>/<code>targetPosition</code> the canvas sets on it.</p><p><code>layoutWithDagre</code> caches positions by topology: the direction, each node&#39;s id and group, and the edges. Switching the duration attribution (same topology, new node data) or returning to a node filter or scope already laid out reads the positions back instead of running dagre again, which costs about 1ms per node. The cache holds the six most recently used layouts, and a layout that throws is not cached.</p><p>Dagre orders the segment groups of a compound graph with a constraint graph built from the order it gave each rank. Three groups whose rank ranges overlap can form a cycle in it, and Dagre then drops those groups from the next sweep, leaving two nodes on one (rank, order) slot and throwing &quot;Not possible to find intersection inside of the rectangle&quot;. <code>runDagre</code> orders through <code>orderGroups</code>: when nodes share a slot it orders again with every pair of groups that share a rank pinned to one fixed left-to-right order. Dagre can still drop nodes then, because the constraints it adds from the observed order can contradict the fixed one, so <code>separateSharedSlots</code> moves any nodes left on one slot to consecutive slots. A graph with no shared slot is ordered exactly as by default.</p><p>A cold layout of a plan with 200 or more nodes runs in a Web Worker (<code>layout.worker.ts</code>, started by <code>layout-worker-client.ts</code>) so the page stays responsive while Dagre works. <code>PlanGraphCanvas</code> asks <code>useLayoutPrewarm</code> for the layouts it needs; the hook sends each uncached one to the worker as plain data (<code>pendingLayoutJob</code>), stores the reply with <code>storeLayout</code>, and the canvas then calls <code>layoutWithDagre</code>, which finds the positions in the cache. The worker runs the same <code>runDagre</code> as the main thread, so the positions are identical. While the layout runs, the first open shows <code>LayoutProgress</code>: an indeterminate progress bar under &quot;Laying out plan graph…&quot;, centered in the canvas. A later relayout keeps the previous graph on screen and shows the same bar in a banner. Smaller graphs, an environment without Web Workers (the single-file export is opened from <code>file://</code>), a worker that fails to start or dies, and a layout that throws all take the main-thread path. The route starts the worker as soon as the expand guardrail dialog opens.</p><p>The canvas mounts only after the layout lands, and its first fit is the same full overview as the rail&#39;s &quot;Fit to view&quot; (<code>FIT_TO_VIEW_OPTIONS</code>). Every automatic re-fit after that (a scope switch, a resize) goes through <code>useReadableFit</code>, which clamps the zoom to at least <code>MIN_READABLE_ZOOM</code> (1.1, so the smallest node text, 10px, renders at 11px on screen). A graph too large to fit at that zoom overflows the canvas and is panned instead of shrunk: centered on an axis that fits, anchored to the start (left/top) on an axis that overflows. The rail&#39;s &quot;Fit to view&quot; is the one-click full overview: a plain <code>fitView</code> with no zoom floor, as is the first fit.</p><p><code>PlanGraphNode.tsx</code> renders every node at a fixed <code>NODE_WIDTH × NODE_HEIGHT</code> (220×90, also what dagre lays the graph out around) with <code>truncate</code>/<code>title</code> on every text field. An operator with an unusually long label/detail/metric string can&#39;t inflate the box and overlap neighbors; it ellipsizes instead, full text on hover.</p><p>Every raw <code>Exchange</code>/<code>BroadcastExchange</code> plan node is split by <code>resolvePlanTree</code> (<code>packages/core/src/event-handlers.ts</code>) into paired write/read halves sharing a <code>sourceNodeId</code> once flattened into the graph. <code>ReusedExchange</code> is classified as an exchange for display purposes but is never split: it carries no <code>exchangeRole</code> and stays a single node. The write half is grouped with its children&#39;s producer component; the read half is grouped with its parent&#39;s consumer component. Both halves get their own entry in <code>buildDurationMap</code>&#39;s duration share. The read half has no metrics of its own and no same-segment descendants, so its share is 0 in exclusive and inclusive mode alike, and <code>PlanGraphNode.tsx</code> shows it as a 0% heat bar next to a &quot;paired: see write half&quot; note. The canvas leaves the share out of the read half&#39;s accessible label.</p><p>Default scope is <code>segment</code>: one Exchange-bounded slice of the plan, resolved via <code>computeSegments</code>/<code>zipSegmentsToStages</code> (<code>packages/core/src/plan-duration-attribution.ts</code>, shared with <code>attributeStageDurationToPlan</code>). <code>computeSegments</code> gives every connected component a stable pre-order identity plus separate parent/depth/traversal metadata. The numeric identity does not encode Exchange-ancestor depth: <code>zipSegmentsToStages</code> orders by the explicit depth metadata (deepest first, plan traversal order for ties), and the display-only fallback uses component-tree edge distance to find the nearest strictly paired component.</p><p>The view has an opt-in &quot;Expand to full plan&quot; toggle gated by a 300-node confirmation dialog (<code>ExpandConfirmDialog.tsx</code>). A segment-lookup failure that would otherwise render an unguarded full plan is routed through the same guardrail rather than bypassing it. That same failure can also resolve to a full-scope model <em>below</em> the guardrail threshold, rendering immediately with no dialog and no <code>requestedScope</code> change. Since it&#39;s a permanent property of that stage&#39;s plan (<code>buildPlanGraphModel</code> is deterministic per <code>(planTree, stageId, appModel)</code>), there is no segment view left for that stage to switch back to. So the toggle, still correctly labeled &quot;Back to segment view&quot; per <code>model?.scope</code>, renders <code>disabled</code> rather than silently no-opping on click. Only the explicit-expand path (<code>requestedScope === &#39;full&#39;</code>) leaves it enabled.</p><p>An oversized fallback or an oversized <code>initialScope: &#39;full&#39;</code> open withholds the graph model until the dialog is confirmed. Dismissing the dialog closes it but does not lift that block, and a dismissed fallback does not reopen the dialog, because the stage has no ungated segment view to show instead.</p><p>The six Plan Advisor finding types (<code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>nestedLoopJoin</code>, <code>pythonUdf</code>, and <code>broadcastSizing</code>&#39;s <code>overBroadcast</code>/<code>underBroadcast</code>, in <code>packages/core/src/detectors.ts</code>) set <code>Finding.planNodeIds</code>, an unambiguous pointer to the specific plan-tree node(s) each finding is about (every node of each repeated subtree occurrence for <code>duplicatePlanSubtree</code>, the flagged scan/write node for <code>smallFiles</code>, the join node for <code>nestedLoopJoin</code>, the <code>BatchEvalPython</code> nodes for <code>pythonUdf</code>, the shuffle <code>data size</code> Exchange node under each of the two join inputs for <code>underBroadcast</code>, and the BroadcastExchange node for <code>overBroadcast</code>). <code>buildPlanGraphModel</code> indexes <code>findings</code> by <code>planNodeIds</code> and attaches each node&#39;s matches to its <code>PlanGraphNodeData.findings</code>, scoped to the current SQL execution (see below), and <code>PlanGraphNode.tsx</code> renders a corner badge from that per-node list. The badge follows the same dot+tag problem-flagging vocabulary as the rest of the dashboard: an <code>ImpactDot</code> plus the ALL-CAPS <code>typeTag</code> (<code>PLAN</code> for every current plan-node finding), colored by the worst band across the node&#39;s findings (critical &gt; warning &gt; info, via the local <code>worstFinding</code> helper), plus a count when the node carries more than one finding. Hovering or focusing the badge opens a tooltip listing each finding by its <code>findingActionLabel</code> (e.g. &quot;Dedupe repeated subtree&quot;, &quot;Compact small files&quot;), so the node discloses which findings hit it without leaving the graph. Every other finding type has no plan-node pointer and renders at the stage level only, via <code>Finding.stageId</code>/<code>Finding.stageIds</code>.</p><p><code>resolvePlanTree</code> prefixes every plan-node id with its owning SQL execution (<code>e&lt;executionId&gt;:n0</code>, <code>e&lt;executionId&gt;:n1</code>, ...), so ids are unique across executions. <code>buildPlanGraphModel</code> also filters <code>findings</code> to <code>finding.executionId === sqlExecutionId</code> (the execution the stage being graphed belongs to) before indexing by node id, as defense in depth: a hand-built or stale finding whose <code>planNodeIds</code> collide with this tree&#39;s ids still can&#39;t badge onto the wrong tree.</p><p>Two other per-node UI features render independently of findings:</p><ul><li>A traffic-light <strong>duration heat bar</strong> on each node (<code>plan-graph-heat.ts</code>&#39;s <code>heatBand</code>, consumed by <code>PlanGraphNode.tsx</code>) bands the node&#39;s duration share into critical/warning/info, reusing the impact-band color tokens (see the Plan Violet exception noted in <code>DESIGN.md</code>).</li><li>A <strong>duration-attribution mode toggle</strong> (<code>PlanGraphDurationModeControl.tsx</code>) switches every node&#39;s displayed share between &quot;Node only&quot; (exclusive) and &quot;Node + descendants&quot; (inclusive), threaded into the model-build/cache key as <code>durationMode</code>.</li></ul><p>One more identity subtlety: a split <code>Exchange</code>/<code>BroadcastExchange</code> pair (the write/read halves <code>resolvePlanTree</code> synthesizes, see above) counts as <strong>one</strong> node for <code>duplicatePlanSubtree</code>&#39;s subtree-size and occurrence counting. <code>computePlanShapes</code> in <code>detectors.ts</code> walks straight through the read half to the write half&#39;s real children, so every real Exchange in a matched subtree is counted once instead of twice.</p><p>The segment-level group box (<code>PlanGraphSegmentGroupNode.tsx</code>) always renders, even in the default single-stage view where it&#39;s the only box on screen. It is headered with its stage id (via <code>segmentStageIds</code>) and a duration chip, or an em-dash placeholder when the segment has no attributed duration. It also carries one finding chip (<code>PlanGraphFindingChip</code>) per finding on its zipped stage, in both the default and the expanded view. Each chip is a <code>TagBadge</code> (the ALL-CAPS tag) followed by the finding&#39;s compact magnitude and recoverable-time detail from <code>formatFindingChipDetail</code> (e.g. &quot;SPILL 4.2 GB · ~38s&quot;: the finding&#39;s own <code>value</code>/<code>metric</code> and its <code>impactEstimate.wallClock</code>), or the bare tag when the finding carries neither.</p><p>The expanded full-plan view draws two nested layers of background group boxes (Dagre <code>compound: true</code> for spacing, <code>computeGroupBounds</code> for both): the same inner segment box described above, plus an outer solid tinted container per distinct stage id (<code>PlanGraphStageGroupNode.tsx</code>, layered under the inner segment box, which is in turn under the plan nodes; all three sit above the edge layer so a routed edge can&#39;t paint over a box&#39;s finding chips) merging every segment zipped to that stage, so a stage split across several Exchange-bounded segments still reads as one unit of work. In this expanded view the segment box still carries its stage&#39;s finding chips inline in its header row. The outer stage box paints its own copy only as a fallback, for a stage no segment box is zipped to (for example, the focal stage when it lost the segment/stage pairing), so every finding appears exactly once. Chips match a stage through <code>Finding.stageId</code> or <code>Finding.stageIds</code>; the default segment scope uses the same matching for its displayed stage. Clicking an outer stage box (&quot;Focus stage N&quot;) switches the route to that stage&#39;s segment view (<code>handleSelectStage</code> in <code>PlanGraphRoute.tsx</code>), with no reopen.</p><p>The outer layer&#39;s own corner tag is positioned opposite the segment box&#39;s top-left header, because a stage with just one segment (the common case) would otherwise show the same &quot;Stage N&quot; header twice, stacked. <code>STAGE_GROUP_PADDING_Y</code> (64, no reserved header height) is sized to strictly exceed the segment layer&#39;s own reserved offset (<code>GROUP_PADDING + GROUP_HEADER_HEIGHT</code> = 56 at the top), while <code>STAGE_GROUP_PADDING_X</code> (32) is kept close to <code>GROUP_PADDING</code>&#39;s own 24px floor so the outer box doesn&#39;t visibly balloon sideways. Both still strictly contain the outer box around its nested segment box(es).</p><p>Every global control lives on one <strong>vertical control rail</strong> down the left edge (<code>PlanGraphControlRail.tsx</code>), grouped View / Navigate / Display, so nothing floats in its own corner. View is zoom in/out and fit (in place of React Flow&#39;s <code>Controls</code>); Navigate is &quot;Next worst duration&quot; and &quot;Next problem&quot; (cycling to the worst-share node and the worst-finding stage, via <code>setCenter</code>); Display is the node-filter/duration Settings popover (<code>PlanGraphSettingsControl</code> with its <code>iconOnly</code> rail variant), plus the legend and minimap toggles. The rail renders inside <code>ReactFlowProvider</code> alongside <code>&lt;ReactFlow&gt;</code>, so its <code>useReactFlow</code> zoom/center calls drive the same instance.</p><p>Clicking a node opens the <strong>detail inspector</strong> (<code>PlanGraphNodeDetail.tsx</code>), a right-docked panel (not a floating card) that reflows the graph rather than covering it. Below Tailwind&#39;s <code>sm</code> breakpoint (640px, <code>useNarrowViewport.ts</code>) it is a bottom sheet about 40 percent tall instead, with a grabber that dismisses it on a downward swipe; while it is open the canvas pans the selected node to the middle of the remaining pane and hides the legend and minimap. Each node box is a fixed size and truncates every field to one line, showing a single <code>primaryMetric</code>; the inspector is where the whole operator is legible: category and segment, duration share, the node&#39;s findings (dot + tag + <code>findingActionLabel</code>), the operator&#39;s <strong>full metric set</strong> (<code>PlanGraphNodeData.metrics</code>, every metric formatted through <code>formatPlanMetricValue</code>), and the <strong>complete plan text</strong> (<code>PlanGraphNodeData.detailText</code>, the untruncated <code>PlanNode.detail</code>). For a split Exchange half the inspector adds an <strong>Exchange section</strong>: which half is in view, the shuffle volume across the boundary (<code>PlanGraphNodeData.exchangeShuffleBytes</code>, the same producing-stage bytes the pairing edge is weighted by, mirrored onto both halves so it shows whichever half is open, or &quot;Broadcast, no shuffle&quot;), and a <strong>jump to the paired half</strong> (<code>PlanGraphNodeData.pairedNodeId</code>). Because the two halves always sit in different segments, the jump selects the partner directly when it is already on screen (full plan) and otherwise expands to the full plan first, then selects it (<code>handleJumpToPaired</code> in <code>PlanGraphRoute.tsx</code>, carried past the selection-reset effect by a pending-jump state); either way the canvas recenters on it via a <code>centerRequest</code> token. The selected node id lives in <code>PlanGraphRoute.tsx</code>, not the canvas, so the route&#39;s one Escape handler closes the inspector first and the whole route only on a second press. Nodes are <code>draggable: false</code> (a read-only, auto-laid-out graph); they click to inspect but don&#39;t move.</p><p>The remaining legibility aids sit on the canvas itself:</p><ul><li>The <strong>MiniMap</strong> (bottom-right) shows by default only while part of the graph is off screen; a click on the rail&#39;s toggle forces it on or off from then on, and the toggle&#39;s pressed state always matches whether it is on screen. It colors each node by the worst finding band on it (<code>planGraphMiniMapNodeColor</code>, <code>plan-graph-minimap.ts</code>), so the overview shows where the problems are; a node with no finding keeps the neutral plan color and the large group boxes recede into a muted fill.</li><li>A <strong>legend</strong> panel (<code>PlanGraphLegend.tsx</code>, collapsed by default, toggled from the rail) keys the operator icons, the heat-bar colors, the shuffle-weighted edge thickness, and the segment-vs-stage box layers.</li><li>When the category filter hides every operator in view, a <strong>status hint</strong> (top-center) names the count hidden and points at Settings, instead of leaving only empty group boxes on screen.</li></ul><p>The view is reached from a &quot;View plan graph&quot; button in <code>PlanView.tsx</code>&#39;s toolbar (gated by the same <code>if (dot)</code> check described in &quot;Plan DOT serialization&quot; above), which opens the stage&#39;s segment, and from the topbar&#39;s graph-view control, which opens the full plan (<code>initialScope: &#39;full&#39;</code>, still behind the 300-node guardrail). Both set a <code>planGraph: { active, stageId, initialScope }</code> Zustand slice (<code>openPlanGraph</code>/<code>closePlanGraph</code>, <code>src/store/store.ts</code>) driving a top-level <code>AppRoutes</code> branch in <code>src/App.tsx</code>, modeled on the <code>comparison</code>/<code>RunComparisonRoute</code> full-takeover pattern.</p><p><code>buildPlanGraphModel</code>&#39;s output is memoized per <code>(activeFileId, stageId, scope, durationMode)</code> in <code>PlanGraphRoute.tsx</code>, since <code>stageId</code> alone isn&#39;t unique across loaded runs and <code>applySnapshot</code> mutates <code>appModel</code> in place rather than replacing it (see <a href="./../state-and-history.html#state-model">State model</a>). The memo cache is a module-level <code>Map</code>, so it survives across route open/close: re-opening the same stage in the same run reuses the cached model instead of rebuilding it. It must still be evicted on a fresh parse or reload. <code>resetModel()</code> (<code>store.ts</code>) bumps a <code>modelResetCount</code> counter for exactly this purpose, and <code>PlanGraphRoute.tsx</code> subscribes to it to clear the cache. A counter rather than a direct call, since <code>store.ts</code> has no view-layer imports anywhere else and importing a <code>.tsx</code> module there would invert that dependency direction.</p><h2 id="duration-share" tabindex="-1">Duration share <a class="header-anchor" href="#duration-share" aria-label="Permalink to &quot;Duration share&quot;">​</a></h2><p><code>exclusiveDurationShare</code> is a non-overlapping partition of each stage&#39;s wall time. The mode-selected <code>durationShare</code> equals it in exclusive mode but adds every descendant&#39;s time into each ancestor in inclusive mode. <code>PlanGraphCanvas</code> therefore divides each node&#39;s share by the sum of the exclusive shares, and labels a segment box with the exclusive sum, so neither number grows with tree depth when the mode flips. A hand-built node without the exclusive field falls back to <code>durationShare</code>.</p><h2 id="layout-ordering-and-cost" tabindex="-1">Layout ordering and cost <a class="header-anchor" href="#layout-ordering-and-cost" aria-label="Permalink to &quot;Layout ordering and cost&quot;">​</a></h2><p><code>runDagre</code> uses the <code>tight-tree</code> ranker rather than Dagre&#39;s default <code>network-simplex</code>: a plan is a tree plus the Exchange read/write split, so <code>tight-tree</code> assigns the ranks a tree wants at a fraction of the cost on a compound graph. <code>longest-path</code> is faster still but places nodes degenerately. <code>nodesep</code> in <code>RL</code> is <code>2 * STAGE_GROUP_PADDING_Y</code>, so two vertically stacked stage boxes (each reserving that padding above and below its members) cannot overlap; <code>BT</code> swaps the axes, with <code>ranksep</code> at that floor and <code>nodesep</code> at <code>2 * STAGE_GROUP_PADDING_X</code> plus a gap. The compound-graph ordering fallback (<code>orderGroups</code>) is described above.</p><p>Calibration data for the layout cost:</p><table tabindex="0"><thead><tr><th>Measure</th><th>Value</th></tr></thead><tbody><tr><td>Compound layout of a 400-node plan, <code>network-simplex</code></td><td>about 1,600 ms</td></tr><tr><td>Same plan, <code>tight-tree</code></td><td>about 500 ms</td></tr><tr><td>Dagre pass cost</td><td>about 1 ms per node</td></tr><tr><td>Stage boxes overlapped without the <code>nodesep</code> floor</td><td>by about 48 px</td></tr><tr><td>Worker threshold (<code>WORKER_LAYOUT_MIN_NODES</code>)</td><td>200 nodes</td></tr></tbody></table>',34)])])}const u=o(n,[["render",s]]);export{g as __pageData,u as default};

package/public/docs/assets/{contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js → contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.lean.js}

@@ -1 +1 @@
-import{_ as o,o as t,c as a,a5 as d}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Plan graph view","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/drill-down/plan-graph.md","filePath":"contributor-guide/architecture/drill-down/plan-graph.md"}'),n={name:"contributor-guide/architecture/drill-down/plan-graph.md"};function s(i,e,r,c,l,h){return t(),a("div",null,[...e[0]||(e[0]=[d("",22)])])}const u=o(n,[["render",s]]);export{g as __pageData,u as default};
+import{_ as o,o as t,c as a,a5 as d}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Plan graph view","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/drill-down/plan-graph.md","filePath":"contributor-guide/architecture/drill-down/plan-graph.md"}'),n={name:"contributor-guide/architecture/drill-down/plan-graph.md"};function s(r,e,i,c,l,h){return t(),a("div",null,[...e[0]||(e[0]=[d("",34)])])}const u=o(n,[["render",s]]);export{g as __pageData,u as default};

package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.K71YB-Rr.js

@@ -0,0 +1 @@
+import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Impact estimation","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/impact-estimation.md","filePath":"contributor-guide/architecture/impact-estimation.md"}'),c={name:"contributor-guide/architecture/impact-estimation.md"};function d(i,e,r,n,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s('<h1 id="impact-estimation" tabindex="-1">Impact estimation <a class="header-anchor" href="#impact-estimation" aria-label="Permalink to &quot;Impact estimation&quot;">​</a></h1><p>Every finding covered by this section carries an optional <code>impactEstimate: {basis, wallClock, estimateMethod, rawWaste?, coreTimeMs?, idleCoreTimeMs?}</code> (<code>packages/core/src/types.ts</code>), attached by its <code>DETECTORS</code> entry&#39;s <code>estimate()</code> (<code>packages/core/src/detectors.ts</code>), which <code>estimateImpact()</code> (<code>packages/core/src/impact-estimator.ts</code>) runs as a post-pass once detection and suppression finish (<code>packages/core/src/analyzer.ts</code>). The waste models those methods compose (assumed throughputs, per-stage measurements, the occupancy clip wrappers) live in <code>packages/core/src/impact-model.ts</code>, and <code>analyze()</code> builds the one <code>EstimateCtx</code> (<code>stages</code>, <code>occupancy</code>, <code>totalCores</code>) every <code>estimate()</code> and every detector&#39;s runtime floor reads. <code>estimateMethod</code> (&#39;measured&#39; | &#39;modeled&#39; | &#39;none&#39;) is a distinct axis from the per-finding <code>confidence</code> field (<a href="./board-widgets.html#confidence-metadata">Confidence metadata</a>): <code>confidence</code> says how much to trust the finding itself, <code>estimateMethod</code> says how its impact number was derived. <code>&#39;none&#39;</code> marks a purely informational finding with no waste model at all (<code>configAudit</code>, <code>stageFailed</code>, <code>failures</code>, <code>incompleteRun</code>, <code>slowHost</code>&#39;s byte-dimension multiDim shapes, <code>gc</code>&#39;s low-GC direction, <code>cacheUtilization</code>&#39;s <code>storageUnobserved</code> caveat, <code>duplicatePlanSubtree</code> when its repeats differ or map to no stage): it&#39;s distinct from <code>&#39;measured&#39;</code>/<code>&#39;modeled&#39;</code>, which both attach a real (if approximate) formula. <code>basis</code> is one of:</p><ul><li><code>&#39;serial&#39;</code>: the tied stage ran (effectively) alone; <code>wallClock</code> is a near-point estimate, <code>low === high</code>.</li><li><code>&#39;contended&#39;</code>: the tied stage shared wall-clock time with others; <code>wallClock</code> is an honest range, <code>high</code> optimistic (assumes the fix could still fully land), <code>low</code> the guaranteed floor.</li><li><code>&#39;resourceOnly&#39;</code>: no wall-clock claim is defensible (not stage-tied by nature, or the stage was excluded from the occupancy sweep), <code>wallClock: null</code>, but the formula&#39;s real signal survives in <code>rawWaste</code>.</li><li><code>&#39;informational&#39;</code>: no quantifiable magnitude at all, <code>wallClock: null</code>, no <code>rawWaste</code>.</li></ul><p><code>rawWaste</code> is not exclusive to <code>resourceOnly</code>/<code>informational</code> findings: every <code>serial</code>/ <code>contended</code> finding carries it too (the only exceptions are <code>coldStart</code>, whose <code>wallClock</code> figure is already unclipped, and <code>estimateMethod: &#39;none&#39;</code> findings, which have no formula at all), holding the formula&#39;s pre-clip magnitude in the formula&#39;s own natural unit (ms, bytes or core-ms). <code>wallClock</code> is what the occupancy model says is recoverable, which clips against the stage&#39;s own physical floor; <code>rawWaste</code> is what the stage really wasted either way. The two answer different questions.</p><p><code>coreTimeMs</code> is the busy core time the fix removes, the executor task time, as <code>{low, high}</code> core-milliseconds with <code>low === high</code>, or <code>null</code> when the detector measures none (never 0 for unknown). <code>estimateImpact()</code> sets it for every estimate through <code>coreTimeFor()</code> in <code>packages/core/src/impact-model.ts</code>, and only from a measured figure: skew and straggler&#39;s removed task time (<code>tailClaimImpact()</code> sets it from the claim&#39;s <code>removedCoreWorkMs</code>), or the raw figure of a type in <code>MEASURED_CORE_TIME_FIGURE</code>: gc&#39;s <code>jvmGCTime</code> (<code>coreMs</code>) and the cross-task executor-time <code>ms</code> sums of <code>retryWaste</code> and <code>speculationWaste</code>. These are set even when the log has no executor cores. A wall-clock claim is never converted to core time, so every finding with only a wall-clock claim gets <code>null</code>. A modeled figure, one that rests on an assumed constant, gets <code>null</code> too: coreLocality&#39;s <code>coreMs</code> (non-local tasks × <code>NETWORK_FETCH_PENALTY_MS</code>) and the <code>coreHours</code> of <code>autoscalingChurn</code> and <code>jobFailureRate</code>. <code>estimateMethod</code> does not decide this, because it describes the wall-clock figure: gc&#39;s is <code>modeled</code> while its <code>jvmGCTime</code> is read from the log. <code>executorCpuTime</code> is never read because it leaves out Python worker CPU.</p><p>A modeled figure on a serial basis carries no core time either. <code>partitionSizing</code>&#39;s <code>lowShuffleParallelism</code> reports <code>rawWaste</code> in <code>ms</code>: stage wall-clock that more partitions shorten, not task work they remove, so it is ranked on <code>wallClock</code>.</p><p>Every surface reads <code>coreTimeMs</code> and <code>remediation</code> from the one <code>analyze()</code> result: the dashboard stores it, the HTML export ships it in its <code>catalog</code>, and the CLI report and MCP tools put it in their finding rows through <code>buildEvidenceReport()</code>. <code>surface-parity.test.js</code> compares the four on the public corpus logs.</p><p>Every surface formats savings through <code>packages/core/src/impact-format.ts</code>. A figure that rounds to zero (&quot;0s&quot;, &quot;0.0 core-h&quot;) reads as a measured nothing, so each formatter shows no figure for it.</p><p><code>coreTimeMs</code> is busy time only. A <code>coreMs</code>/<code>coreHours</code> raw figure that counts allocated capacity no task ran on carries <code>rawWaste.idle: true</code>: <code>utilization</code> (<code>coreHours</code>) and <code>stageShape</code>&#39;s <code>lowParallelism</code> and <code>taskStageSkew</code> (<code>coreMs</code>). <code>coreTimeFor()</code> gives those <code>null</code>, and <code>rawWasteMeaning()</code> in <code>packages/core/src/impact-format.ts</code> labels them &quot;of idle core capacity&quot; instead of &quot;of core time&quot; on every surface. <code>memoryUtilization</code>&#39;s <code>idleCores</code> counts <code>mbSeconds</code>, which has no core time either. <code>utilization</code> also reports the idle figure as <code>idleCoreTimeMs</code> (<code>{low, high}</code> core-milliseconds, <code>low === high</code>): <code>allocatedCoreMs - busy core-ms</code>, which is <code>rawWaste</code> in core-ms, set by its <code>estimate()</code> and never read by <code>coreTimeFor()</code>. The capacity is <code>allocatedCoreMs()</code> in <code>packages/core/src/allocation.ts</code>, the figure behind <code>metrics.allocation.coreHours</code>. <code>idleCores</code> is the memory view of the same condition and has none. A stage&#39;s slow tail is also counted once: <code>skew</code> and <code>straggler</code> both measure the task time removed from it, so on one stage <code>skew</code> carries it and <code>straggler</code> has <code>null</code> (<code>countTailCoreTimeOnce()</code> in <code>packages/core/src/impact-estimator.ts</code>).</p><h2 id="occupancy-weighted-attribution" tabindex="-1">Occupancy-weighted attribution <a class="header-anchor" href="#occupancy-weighted-attribution" aria-label="Permalink to &quot;Occupancy-weighted attribution&quot;">​</a></h2><p><code>packages/core/src/occupancy.ts</code> sweeps every stage&#39;s observed <code>[submittedAt, completedAt)</code> window and splits each instant&#39;s wall-clock among concurrently-active stages proportional to <code>coreWeight(S) = stage.executorRunTime / stageDurationMs</code> (an average-concurrency proxy, held constant across the stage&#39;s whole window: this codebase has no per-task timestamps outside the parser worker to do better). Summing a stage&#39;s share across its own window gives its <code>occupancy(S)</code>; <code>gate(S) = occupancy(S) / duration(S) ∈ [0, 1]</code> is the single number the estimate reads. 1.0 means the stage ran completely alone; 0 means the stage had zero <code>executorRunTime</code> while overlapping other, positive-weight stages, so it got no share of the shared window. When every stage active over an interval has zero weight (no <code>executorRunTime</code>), that interval is split equally among them instead, so a stage that ran alone still gets <code>gate</code> 1. Stages with <code>duration(S) &lt;= 0</code> (Spark-skipped stages, or a malformed <code>submittedAt === completedAt</code>) are excluded from the sweep entirely.</p><p><code>efficiency-model.ts</code> reports <code>floorZeroSkewMs</code> (total task time / peak concurrent cores) as its theoretical floor.</p><p><code>ceiling(S) = max(stage.taskDurationMax, stage.executorRunTime / totalCores)</code> is a physical floor on a stage&#39;s own duration: bounded below by its single longest task (unsplittable no matter how much parallelism exists) or by its core-work spread across every core in the cluster, whichever is larger. Every waste formula&#39;s raw claim is clipped against it before gate-weighting: <code>wasteMs_clipped(S) = min(wasteMs_claimed, max(0, duration(S) - ceiling(S)))</code>, so a finding can never claim to save more than the portion of the stage&#39;s observed duration that sits above its own unbeatable floor.</p><p><code>skew</code> and <code>straggler</code> are the exception (<code>estimateSingleStage</code>&#39;s <code>shortensLongestTask</code> option, passed by <code>detectors.ts</code>&#39;s <code>tailClaimImpact</code>, which both their <code>estimate()</code> and their <code>detect()</code> runtime-floor gate call on the same tail claim, so firing and display agree; <code>stageSlowness</code>&#39;s more-partitions estimate and <code>partitionSizing</code>&#39;s <code>lowShuffleParallelism</code> pass it too, since splitting partitions splits the longest task). Their claim shortens the stage&#39;s longest task itself, so <code>taskDurationMax</code> can&#39;t be their floor: clipping against it would cap a stage gated by one straggler at <code>duration(S) − taskDurationMax</code>, about zero, exactly when the fix recovers the most. Their floor is instead the longest task the fix leaves plus the core work the fix leaves, <code>max(taskDurationMax − wasteMs_claimed, longestTaskAfterFixMs, (stage.executorRunTime − removed) / totalCores)</code>, where <code>removed</code> (<code>tailRemovedWorkMs</code>) is the larger of the finding&#39;s single-task delta (see below) and <code>stragglerExcessMs</code>, and <code>longestTaskAfterFixMs</code> is the longest task the fix leaves for <code>skew</code> and <code>straggler</code> (see below; for <code>lowShuffleParallelism</code> it is <code>taskDurationMax × taskCount / targetTaskCount</code>, the longest task after an even split into the target task count, a modeled figure). Counting the stragglers&#39; own run time as work the stage can&#39;t shed would floor a stage whose tail is most of its core time near its observed duration.</p><p>The claim itself is <code>tailRecoveryMs</code> (<code>occupancy.ts</code>), which returns the stage&#39;s <code>tailReplayRecoveryMs</code>: <code>finalizeStage</code> replays the stage&#39;s own tasks with list scheduling (<code>computeTailReplayRecoveryMs</code> in <code>stage-quantiles.ts</code>: slots = <code>peakConcurrentTasks</code>, at least 1 and at most the task count; tasks in launch order, equal launch times in task-array order, each on the slot that frees first) once with the real durations and once with every task over 4× P50 capped at P50, and takes the difference. It runs only on stages with a task over 4× P50 (every other stage recovers 0) and costs a launch-order index and a slot heap per stage: 5-9 ms over the 236 such stages (23,641 tasks) of the largest real log, whose parse takes 5.5-6.9 s. <code>dev/eval-tail-replay.mjs</code> keeps an independent copy of the replay as its ground truth, and the two agree on every tail stage of the real logs and the corpus.</p><p>A stage without the field (one built by hand, as in detector tests) falls back to a stage-level estimate: the larger of the single-task delta above and <code>stragglerExcessMs / peakConcurrentTasks</code>, the summed excess over P50 of every task slower than 4× P50, spread over the most tasks the stage ever ran at once. That estimate assumes the tail either gates the stage through one task or spreads evenly over the slots; the replay knows when each slow task launched. Stragglers clustered at the end of a stage recover more than the even spread (one real stage: 12 of 200 tasks over 4× P50 on 29 slots, 89.0s estimated, 127.3s replayed), a long task launched early overlaps the rest of the stage and recovers less than its excess (8.4s estimated, 3.0s replayed), and a speculation-driven stage with no task over 4× P50 recovers nothing (5.8s estimated on two real stages).</p><p>The single-task deltas set the removed work above and the fallback estimate. <code>straggler</code>&#39;s runs from <code>taskDurationMax</code> down to the longest task its fix leaves (<code>stragglerFixLongestTaskMs</code>), not down to P50: the fix brings every task over 4× P50 to the median, so the stage still waits on its longest task at or under that (<code>longestNonStragglerMs</code>, from <code>finalizeStage</code>), and that task is also passed as a floor (<code>longestTaskAfterFixMs</code>). A speculation-driven finding with no task over 4× P50 keeps the P50 delta. <code>skew</code> keeps its P50 delta, since repartitioning may even out tasks under 4× P50 too, but takes the same <code>longestTaskAfterFixMs</code> floor: without it a 100 s stage whose next-longest task ran 39 s would have skew claiming 90 s where straggler claims 61 s.</p><p><code>analyzer.ts</code> feeds this <code>totalCores</code> from <code>packages/core/src/core-count.ts</code>&#39;s <code>computePeakConcurrentCores(app, executorsAdded, executorsRemoved)</code> (the same peak concurrent cores <code>efficiency-model.ts</code>&#39;s floor, <code>wasted-core-hours.ts</code> and the <code>utilization</code> finding report as the cluster size), not <code>computeTotalCores</code>, which only <code>scaling-sim.ts</code> uses. <code>computeTotalCores</code> sums every <code>ExecutorAdded</code> event&#39;s cores regardless of overlap, so under dynamic allocation or executor replacement it can far exceed the cores ever actually concurrent, which understates <code>ceiling(S)</code> and lets churn inflate a finding&#39;s claimed wall-clock. <code>computePeakConcurrentCores</code> instead sweeps add/remove events by timestamp and tracks the running total&#39;s peak, so a churned-through executor&#39;s cores are never double-counted against its replacement&#39;s. Same-timestamp events tie-break by delta ascending, so a removal applies before a same-instant replacement&#39;s addition (otherwise a same-instant swap would momentarily double-count both as concurrent). If every <code>executorsAdded</code> entry lacks <code>totalCores</code> the cores sweep peaks at zero and tells us nothing; the function then falls back to sweeping peak <em>executor count</em> instead (still concurrency-aware, just cores-blind) and multiplies by the configured per-executor core count, rather than falling back to <code>executorsAdded.length × cores</code>, which overcounts under churn.</p><p>Per-finding estimate, using <code>wasteMs_clipped(S)</code>:</p><ul><li><code>gate(S) &gt;= 0.999</code>: <code>basis: &#39;serial&#39;</code>, <code>low = high = wasteMs_clipped(S)</code>.</li><li><code>gate(S) &lt; 0.999</code>: <code>basis: &#39;contended&#39;</code>, <code>high = wasteMs_clipped(S)</code>, <code>low = wasteMs_clipped(S) * gate(S)</code>.</li></ul><p>A finding spanning multiple stages (<code>stageIds</code>, plural) sums each stage&#39;s own estimate and caps the joint total at the union of just that finding&#39;s own stage windows (via <code>mergeIntervals</code>, <code>packages/core/src/intervals.ts</code>): <code>high = min(Σ high_i, unionMs(stageIds))</code>, <code>low = min(Σ low_i, unionMs(stageIds))</code>. This is what prevents overclaiming when two or more of a finding&#39;s stages overlap in wall-clock time: a plain sum-and-cap. The union cap can force <code>low === high</code> numerically even when the constituent stages were individually contended (e.g. two fully-overlapping stages each at <code>gate</code> 0.5), so <code>basis</code> isn&#39;t derived from that numeric equality: a multi-stage finding gets <code>basis: &#39;serial&#39;</code> only when every one of its per-stage estimates was itself <code>&#39;serial&#39;</code>; otherwise <code>&#39;contended&#39;</code>.</p><p>Regression guard: a finding whose stage (or, for a multi-stage finding, every one of its stages) ran alone (<code>gate &gt;= 0.95</code>, deliberately looser than the <code>0.999</code> &quot;serial&quot; cutoff above: this guard exists to catch egregious false zeros, not to gate which <code>basis</code> a finding gets) with a real underlying magnitude (<code>rawWaste.value &gt; 0</code>) must never report <code>wallClock.high === 0</code> or <code>basis: &#39;informational&#39;/&#39;resourceOnly&#39;</code>. Covered by <code>packages/core/test/impact-estimator-real-log.test.js</code> (run with <code>npm run test:core</code>; skipped unless the <code>private-log-01.zstd</code> fixture is in <code>examples/</code>); relies on <code>rawWaste</code> being attached to every serial/contended-capable formula (see above), so it&#39;s blind to <code>coldStart</code>, the purely informational (<code>estimateMethod: &#39;none&#39;</code>) finding types, <code>stageShape</code> (resourceOnly by design) and <code>shuffle</code> findings whose stage measured zero fetch wait (a measured zero, not a clip artifact).</p><p>Real-log spot-check (<code>private-log-01.zstd</code>): median <code>gate</code> across stages was <code>≈0.34</code> (0.3428060791718594 exactly); <code>collectRun</code> plus the occupancy sweep together took <code>≈6,589</code>ms on the largest fixture measured (<code>private-log-03.zstd</code>, <code>1169</code> stages): parse-dominated, the sweep alone was not isolated by this measurement, but not a magnitude that suggests a regression either.</p>',23)])])}const m=o(c,[["render",d]]);export{g as __pageData,m as default};

package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js → contributor-guide_architecture_impact-estimation.md.K71YB-Rr.lean.js}

@@ -1 +1 @@
-import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Impact estimation","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/impact-estimation.md","filePath":"contributor-guide/architecture/impact-estimation.md"}'),c={name:"contributor-guide/architecture/impact-estimation.md"};function d(i,e,r,n,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s("",21)])])}const m=o(c,[["render",d]]);export{g as __pageData,m as default};
+import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Impact estimation","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/impact-estimation.md","filePath":"contributor-guide/architecture/impact-estimation.md"}'),c={name:"contributor-guide/architecture/impact-estimation.md"};function d(i,e,r,n,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s("",23)])])}const m=o(c,[["render",d]]);export{g as __pageData,m as default};

package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as d}from"./chunks/framework.DSg0KOwT.js";const m=JSON.parse('{"title":"Caveats, tuning and coverage","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md","filePath":"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md"}'),s={name:"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md"};function c(r,e,i,n,l,h){return o(),a("div",null,[...e[0]||(e[0]=[d('<h1 id="caveats-tuning-and-coverage" tabindex="-1">Caveats, tuning and coverage <a class="header-anchor" href="#caveats-tuning-and-coverage" aria-label="Permalink to &quot;Caveats, tuning and coverage&quot;">​</a></h1><p>Spot-checks for each estimate formula, how skew and straggler split a tail, the effect of tuned thresholds and which finding types have an estimate.</p><h2 id="per-formula-spot-checks" tabindex="-1">Per-formula spot-checks <a class="header-anchor" href="#per-formula-spot-checks" aria-label="Permalink to &quot;Per-formula spot-checks&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Detector</th><th>Formula basis</th><th>Spot-check</th></tr></thead><tbody><tr><td>gc</td><td><code>jvmGCTime / (executorRunTime / stageDurationMs)</code></td><td><code>private-log-01.zstd</code>, stage 507: <code>jvmGCTime</code>=1080ms, <code>executorRunTime</code>=27509ms, <code>stageDurationMs</code>=56279ms → <code>wasteMs</code> = 1080 / (27509/56279) ≈ 2209.5ms. That&#39;s ≈3.9% of the stage&#39;s 56.3s wall-clock duration, matching the finding&#39;s own reported <code>gcPct</code> (3.9%) exactly, as the formula guarantees by construction. Under the occupancy model this stage&#39;s <code>gate</code> is <code>0.041</code> (0.04145044590332269 exactly): <code>basis: &#39;contended&#39;</code>, <code>wallClock: {low: 91.6, high: 2209.5}</code> (91.5850382272182 / 2209.506706895925 exactly).</td></tr><tr><td>shuffle</td><td><code>shuffleReadBytes / (SHUFFLE_THROUGHPUT_BPS × executors that ran the stage)</code></td><td><code>private-log-02.zstd</code>, stage 99 (<code>SHFL</code> finding): <code>shuffleReadBytes</code>=204,172,518,504 over 8 executors → <code>wasteMs</code> = 204172518504 / (8 × 125,000,000) × 1000 ≈ 204,173ms, against the stage&#39;s 763,776ms duration. The tasks&#39; own measured shuffle fetch wait on this stage is 25.2s of wall-clock (<code>fetchWaitTime</code> / average concurrency), so even the per-link model runs well above the network stall actually observed, and the claim is capped there: 25.2s, <code>measured</code>.</td></tr><tr><td>spill</td><td><code>diskBytesSpilled / (SPILL_IO_THROUGHPUT_BPS × executors that ran the stage)</code></td><td>Same run and stage (99): <code>diskBytesSpilled</code>=145,978,433,675 (note: the <code>SPILL</code> finding&#39;s own <code>value</code>/<code>metric</code> report <code>memoryBytesSpilled</code>=913,686,966,448, ~6x larger; the formula correctly uses the smaller disk figure, not that one) over 8 executors → <code>wasteMs</code> = 145978433675 / (8 × 200,000,000) × 1000 ≈ 91,237ms (91236.521046875 exactly, matching <code>wallClock</code>, below the clip).</td></tr></tbody></table><h2 id="one-tail-one-finding-skew-straggler" tabindex="-1">One tail, one finding: skew / straggler <a class="header-anchor" href="#one-tail-one-finding-skew-straggler" aria-label="Permalink to &quot;One tail, one finding: skew / straggler&quot;">​</a></h2><p><code>skew</code> and <code>straggler</code> both claim the stage&#39;s <code>tailReplayRecoveryMs</code> (see <a href="./../impact-estimation.html#occupancy-weighted-attribution">Occupancy-weighted attribution</a>), whichever skew branch fired. The skew branch (P95 or max over P50) changes only the fallback single-task delta on a stage without the replay field. Which of the two reports a tail follows its cause (<code>tailVerdict</code> in <code>detectors.ts</code>, from the stage&#39;s <code>tailAttribution</code>): <code>skew</code> takes a tail whose extra time follows data volume, and <code>straggler</code> takes the rest, including a tail with no measured cause (<code>cause: &#39;unattributed&#39;</code>) or a stage with no tail attribution. Both judge a tail with skew&#39;s resolved thresholds, so the two never fire on the same stage and their recoverable time is never counted twice. Both are clipped with the post-fix floor described under <a href="./../impact-estimation.html#occupancy-weighted-attribution">Occupancy-weighted attribution</a>, not the plain <code>ceiling</code>, so a stage gated by one dominant outlier task reports that task&#39;s excess as recoverable instead of the near-zero room <code>ceiling &gt;= taskDurationMax</code> would leave. Both detectors also carry a <code>confidence</code> field that scales <code>low</code>/<code>medium</code>/<code>high</code> off how far the finding&#39;s own ratio (skew: <code>ratioWarn</code>) or task share (straggler: <code>shareWarn</code>/<code>warnPct</code>, <code>critPct</code>) sits past its detector threshold (unvalidated; see the confidence-disclosure note in detector-contract.md).</p><p><code>stageShape</code>&#39;s <code>taskStageSkew</code> rule doesn&#39;t participate in this caveat: it reports a <code>resourceOnly</code> idle-core-ms figure (see the coverage table below) instead of a wall-clock claim, so there&#39;s nothing left to double-count against <code>skew</code>/<code>straggler</code>. It fires on the same straggler tail those two price (on the corpus, 9 of its 10 firings share a stage with a <code>skew</code> or <code>straggler</code> finding), which is why it stays off the wall-clock path.</p><h2 id="tuned-thresholds" tabindex="-1">Tuned thresholds <a class="header-anchor" href="#tuned-thresholds" aria-label="Permalink to &quot;Tuned thresholds&quot;">​</a></h2><p>The estimates are calibrated against the default detector thresholds: the spot-checks above, the corpus snapshot and the tail-replay scores (<code>dev/eval-tail-replay.mjs</code>) all measure findings the defaults produce. The formulas read the stage&#39;s own figures, not the thresholds, but a threshold decides which findings exist, so an override that loosens one produces findings, near its new gate, that no estimate here was checked against. When the CLI or the MCP server runs with <code>--thresholds</code> (see <a href="./../detector-contract.html#tuning-thresholds">Tuning thresholds</a>), every finding from a tuned detector carries <code>tunedThresholds</code>, a finding with an estimate figure gets a <code>validationRequired</code> sentence saying that estimate is unvalidated, and the report header lists the tuned detectors. The estimate itself is computed the same way; nothing rescales or drops it. The impact bands (<code>impact-band.ts</code>) keep their default floors on a tuned run, except that a tuned <code>skew</code> or <code>straggler</code> <code>floorPctWarn</code>/<code>floorPctCrit</code> grades that entry&#39;s own findings.</p><h2 id="idle-cpu-share" tabindex="-1">Idle CPU share <a class="header-anchor" href="#idle-cpu-share" aria-label="Permalink to &quot;Idle CPU share&quot;">​</a></h2><p><code>stageSlowness</code> claims nothing for a stage that read no input and no shuffle bytes and whose tasks spent under <code>IDLE_CPU_SHARE_MAX</code> (<code>packages/core/src/impact-model.ts</code>) = 1% of <code>executorRunTime</code> on CPU: they sat waiting on something outside Spark, which more partitions do not split. On the calibration logs every non-Python stage under 1% is a JDBC read, a file listing or a Delta log read, while file writes, which more partitions do parallelize, start at 2%. The share is not used when the log records no CPU time (older Spark) or the stage ran Python through a Python worker (a <code>PythonRDD</code> stage, or a Python UDF operator in its SQL plan): the worker process&#39;s CPU is not in <code>executorCpuTime</code>, which counts the JVM task thread only, so such stages read 0.1% while computing.</p><h2 id="calibration-data" tabindex="-1">Calibration data <a class="header-anchor" href="#calibration-data" aria-label="Permalink to &quot;Calibration data&quot;">​</a></h2><p>Calibrated on 14 real logs. The per-detector floors are in <a href="./../detector-contract/threshold-tables.html#calibration-data">Calibration data</a>; these figures back the models. Single-stage examples sit in the coverage table below.</p><table tabindex="0"><thead><tr><th>Model or floor</th><th>Figure</th></tr></thead><tbody><tr><td>Per-executor I/O parallelism (<code>stageIoParallelism</code>)</td><td>A model that divides a stage&#39;s bytes by one device&#39;s bandwidth claims up to 1,870 s of shuffle time on stages whose tasks measured about 0 s of shuffle fetch wait (222 of 284 shuffle findings)</td></tr><tr><td>Fetch-wait cap on <code>shuffle</code></td><td>The per-executor link model alone totals 2,780 s over 284 shuffle findings and 256 s once capped at the measured fetch wait, with about zero fetch wait on 5 of the 10 non-<code>info</code> findings: reads that overlapped compute stalled nothing</td></tr><tr><td>Retry wall-clock (<code>retryWallClockMs</code>)</td><td>One stage lost 4 attempts of 36.6 s each, all first attempts, while running 41 tasks at once: the summed time is 146.6 s, the longest-chain claim is 36.6 s</td></tr></tbody></table><h2 id="per-finding-type-coverage" tabindex="-1">Per-finding-type coverage <a class="header-anchor" href="#per-finding-type-coverage" aria-label="Permalink to &quot;Per-finding-type coverage&quot;">​</a></h2><p>One row per distinct <code>type</code> string <code>packages/core/src/detectors.ts</code> actually emits: every entry must declare an <code>estimate()</code> (the compiler rejects one without it), so every row below has one, and the table itself is the coverage count, not a number restated here. <code>broadcastSizing</code> is a <code>DETECTORS</code> entry label only, and the plan-walk it drives emits <code>overBroadcast</code>/<code>underBroadcast</code> findings instead, so those two are the rows that appear, not <code>broadcastSizing</code> itself. Tag meanings: <code>measured</code> and <code>modeled</code> both produce a real, gate-clipped, non-<code>{0,0}</code><code>wallClock</code> (the difference is whether the formula&#39;s inputs are recorded per-stage fields or an assumed constant like a throughput figure); <code>cost-only</code> always reports <code>basis: &#39;resourceOnly&#39;</code>, <code>wallClock: null</code> but carries its real signal in <code>rawWaste</code>; <code>informational-only</code> reports <code>basis: &#39;informational&#39;</code>, <code>wallClock: null</code> with no <code>rawWaste</code> at all, since there&#39;s nothing quantifiable. A type with more than one tag fires a different formula per <code>variant</code>/<code>rule</code> on the same finding type; the basis column says which.</p><table tabindex="0"><thead><tr><th>Finding type</th><th>Scope</th><th>Tag</th><th>Basis</th></tr></thead><tbody><tr><td><code>retryWaste</code></td><td>stage</td><td>measured / modeled</td><td><code>retryWasteMs</code> is summed attempt time, and attempts of different tasks run side by side (one lost executor fails every task it was running at once). With every wasted attempt sampled (<code>retryTaskSamples</code>, capped at 20), modeled: the longest retry chain (highest sampled attempt number + 1) times the mean wasted attempt, or <code>retryWasteMs / peakConcurrentTasks</code> when larger, capped at <code>retryWasteMs</code>. Otherwise measured, <code>retryWasteMs</code> itself. Gate-clipped; the summed figure is kept as <code>rawWaste</code> in <code>ms</code>.</td></tr><tr><td><code>speculationWaste</code></td><td>stage</td><td>measured</td><td><code>speculationWasteMs</code>, gate-clipped; pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>coldStart</code></td><td>app</td><td>measured</td><td><code>gapSeconds × 1000</code> (first stage submitted to first executor added), unclipped, <code>basis: &#39;serial&#39;</code> unconditionally (a pre-first-task gap can&#39;t overlap any stage)</td></tr><tr><td><code>gc</code></td><td>stage</td><td>modeled / informational-only</td><td>high-GC: <code>jvmGCTime / (executorRunTime / stageDurationMs)</code>, gate-clipped: the concurrency division is an approximation, not a reconstruction, hence <code>modeled</code>; <code>rawWaste</code> in <code>coreMs</code> is the raw <code>jvmGCTime</code> sum before that conversion. Low-GC (<code>direction: &#39;low&#39;</code>): informational-only, since its fix (less executor memory) raises GC rather than recovering it</td></tr><tr><td><code>skew</code></td><td>stage</td><td>measured</td><td><code>tailRecoveryMs</code>: the stage&#39;s <code>tailReplayRecoveryMs</code> (a task-level replay with every task over 4× P50 capped at P50), gate-clipped against the post-fix floor (<code>shortensLongestTask</code>) with the longest task the fix leaves as a floor (<code>longestTaskAfterFixMs</code>, as for <code>straggler</code>); pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>straggler</code></td><td>stage</td><td>measured</td><td><code>tailRecoveryMs</code>, as for <code>skew</code> (0 on a speculation-driven stage with no task over 4× P50), gate-clipped against the post-fix floor (<code>shortensLongestTask</code>, <code>longestTaskAfterFixMs</code>)</td></tr><tr><td><code>stageShape</code></td><td>stage</td><td>cost-only</td><td><code>&#39;lowParallelism&#39;</code> and <code>&#39;dataExplosion&#39;</code> are <code>estimateMethod: &#39;measured&#39;</code> (real per-stage fields, no assumed constant), <code>&#39;taskStageSkew&#39;</code> is <code>&#39;modeled&#39;</code>: <code>&#39;lowParallelism&#39;</code> → <code>rawWaste</code> in <code>coreMs</code> (idle cores × stage duration); <code>&#39;dataExplosion&#39;</code> → <code>rawWaste</code> in <code>bytes</code> (<code>outputBytes − inputBytes</code>); <code>&#39;taskStageSkew&#39;</code> → <code>rawWaste</code> in <code>coreMs</code> (<code>max(0, min(totalCores, taskCount) − 1) × (taskDurationMax − taskDurationP50)</code>, the cores idle during the straggler&#39;s tail at achieved concurrency, an upper bound that assumes every executor stays allocated through the tail, so it is clipped to the run&#39;s idle core time, <code>EstimateCtx.runIdleCoreMs</code>; <code>totalCores</code> here is the finding&#39;s own figure: the executors that ran the stage × <code>spark.executor.cores</code>, not the run&#39;s peak concurrent cores)</td></tr><tr><td><code>slowHost</code></td><td>stage</td><td>measured / informational-only</td><td>duration-based variants: the slow host&#39;s absolute figure minus the stage median, <code>max(0, hostMeanMs − taskDurationP50)</code> for <code>hostMeanRatio</code>/<code>durationShare</code> and <code>max(0, execMaxValue − taskDurationP50)</code> for <code>multiDim</code>+<code>taskTime</code> (<code>value</code> is a ratio or share, never ms), gate-clipped, pre-clip figure kept as <code>rawWaste</code> in <code>ms</code>; byte-based <code>multiDim</code> dimensions: no formula yet (<code>estimateMethod: &#39;none&#39;</code>). <code>multiDim</code>+<code>taskTime</code> picks the largest of N executor means, so it carries <code>confidence: &#39;low&#39;</code>, a <code>validationRequired</code> sentence on chance, <code>executorCount</code> and <code>taskCount</code>; the band rule and the id are unchanged (the id discriminator <code>host</code> stays unset)</td></tr><tr><td><code>duplicatePlanSubtree</code></td><td>sql</td><td>measured</td><td>per stage in <code>stageShares</code>: its task-active time (<code>taskActiveMs</code>, the union of its task intervals; submit-to-complete only when absent) × the repeated operators&#39; share of that stage × the redundant fraction <code>(occurrences − 1) / occurrences</code>, summed and capped at the union of those stages&#39; spans. A stage shared with other operators (the consuming join, the other join side) contributes only its share, so sibling groups can&#39;t claim one stage twice, and a stage left waiting for cores (2491 s open, 60 s of tasks on a real log) claims only its task time. No claim (<code>informational</code>) when the repeats&#39; details differ (<code>occurrencesIdentical: false</code>) or no repeated operator has a stage (the execution-wide <code>stageIds</code> fallback stays for linking only).</td></tr><tr><td><code>shuffle</code></td><td>stage</td><td>modeled / measured</td><td><code>shuffleReadBytes / (SHUFFLE_THROUGHPUT_BPS × executors)</code> (assumed ~125MB/s per executor link; <code>executors</code> = <code>executorStats.length</code>, the executors that ran the stage, min 1), capped at the fetch wait the tasks measured in wall-clock (<code>fetchWaitTime / (executorRunTime / stage duration)</code>, the <code>gc</code> conversion; <code>measured</code> when that cap binds), gate-clipped; <code>rawWaste</code> in <code>bytes</code> is the measured <code>shuffleReadBytes</code> behind it. The link model can&#39;t see whether reads stalled the tasks, which is why the fetch-wait cap applies. Fetch wait misses disk reads and deserialization, so the cap is a floor on what a shuffle costs, not its total</td></tr><tr><td><code>spill</code></td><td>stage</td><td>modeled</td><td><code>diskBytesSpilled / (SPILL_IO_THROUGHPUT_BPS × executors)</code> (assumed ~200MB/s per executor&#39;s local disk, same executor count as <code>shuffle</code>), gate-clipped; <code>rawWaste</code> in <code>bytes</code> is <code>diskBytesSpilled</code>, which is the number the formula uses and not the <code>memoryBytesSpilled</code> the finding&#39;s own <code>metric</code> displays. <code>assumptions</code> states the rate, the executor count and that memory spill, serialization and GC are not counted</td></tr><tr><td><code>stageSlowness</code></td><td>stage</td><td>modeled</td><td>what more partitions (the finding&#39;s recommendation) could recover: the stage&#39;s task-active time (<code>taskActiveMs</code>, the union of its tasks&#39; launch-to-finish intervals from <code>finalizeStage</code>) × <code>max(0, 1 − taskCount / totalCores)</code>, gate-clipped with the post-fix floor (<code>shortensLongestTask</code>: splitting partitions splits the longest task too). A stage that already ran at least as many tasks as the cluster had cores claims 0, and so does one that read no input and no shuffle bytes and whose tasks spent under 1% of <code>executorRunTime</code> on CPU (<code>executorCpuTime</code>): they sat waiting on something outside Spark, which more partitions don&#39;t split (a 1-task JDBC <code>count</code> stage open 27 minutes on 5s of CPU, 0.33%, claims 0). Non-Python stages under 1% CPU share are JDBC reads, file listings and Delta log reads, and file writes start at 2%. The share is skipped, keeping the claim, when the log records no CPU time (older Spark) or the stage ran Python through <code>PythonRDD</code>, whose worker CPU the JVM metric misses (0.1% on computing stages); a Python UDF inside a SQL stage that reads no bytes isn&#39;t detected and can still read idle. A stage that read input or shuffle bytes keeps its claim whatever its CPU share, so a Python UDF over real input is never zeroed. Time a stage sat open with no task running (queued for slots) claims nothing. No cluster core count: informational.</td></tr><tr><td><code>partitionSizing</code></td><td>stage</td><td>modeled</td><td><code>maxPartitionTooBig</code>/<code>shufflePartitionSkew</code>: shuffle-throughput formulas, gate-clipped. <code>lowShuffleParallelism</code>: stage duration scaled down by the shortfall between actual and ideal-partition-count task counts (<code>stageDurationMs × (1 − taskCount / targetTaskCount)</code>), i.e. the serialized work more partitions would let run concurrently, not the scheduling cost of the tasks you&#39;d add to fix it. The claim shortens the longest task, so it is capped at the reduction of that task (<code>taskDurationMax − longestTaskAfterFixMs</code>, with <code>longestTaskAfterFixMs = taskDurationMax × taskCount / targetTaskCount</code>, the longest task after an even split: a modeled figure, optimistic when <code>shufflePartitionSkew</code> fires on the same stage) and gate-clipped with the post-fix floor (<code>shortensLongestTask</code>). <code>coreTimeMs</code> is null: the figure is wall-clock, not removed task time</td></tr><tr><td><code>tinyTask</code></td><td>stage</td><td>measured / modeled</td><td>excess task count over 10% of the stage&#39;s actual count, × the stage&#39;s own measured per-task overhead (summed task wall time from <code>executorStats</code> minus <code>executorRunTime</code>, over <code>taskCount</code>), ÷ the stage&#39;s achieved task concurrency (task time ÷ stage duration, floored at 1), gate-clipped; <code>measured</code>. A stage with no <code>executorStats</code> or no measurable overhead falls back to the assumed 50ms per task, undivided (<code>modeled</code>). Pre-clip figure kept as <code>rawWaste</code> in <code>ms</code>.</td></tr><tr><td><code>smallFiles</code></td><td>sql</td><td>modeled / cost-only</td><td><code>fileCount × FILE_OPEN_OVERHEAD_MS</code> (10 ms per file), divided for a read by the most tasks its stages ran at once (<code>peakConcurrentTasks</code>: tasks open their files in parallel; 91,344 files claimed 76 s on a 117 s stage that ran 314 tasks at once) and kept serial for a write (the job commit moves each file on the driver). The figure is split evenly across <code>stageIds</code>, each stage&#39;s share is gate-clipped, and the sum is capped at the stages&#39; union; with no <code>stageIds</code> to map to, cost-only with that same figure as <code>rawWaste</code> in <code>ms</code>. <code>stageIds</code> is narrowed the same way (see <a href="./../detector-contract/plan-attribution.html#stage-id-attribution-for-plan-advisor-findings">Stage-ID attribution for Plan Advisor findings</a>); falls back to the whole execution&#39;s stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>overBroadcast</code></td><td>sql</td><td>modeled / cost-only</td><td><code>broadcastBytes / BROADCAST_BANDWIDTH_BPS</code>, summed and capped over <code>stageIds</code>&#39;s union; cost-only with <code>rawWaste</code> in <code>ms</code> when not stage-mappable. <code>stageIds</code> is narrowed the same way; falls back to the whole execution&#39;s stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>underBroadcast</code></td><td>sql</td><td>modeled / cost-only</td><td><code>smallerSideBytes / BROADCAST_BANDWIDTH_BPS</code>, summed and capped over <code>stageIds</code>&#39;s union; cost-only with <code>rawWaste</code> in <code>ms</code> when not stage-mappable. <code>stageIds</code> is narrowed the same way; falls back to the whole execution&#39;s stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>memoryUtilization</code></td><td>app</td><td>cost-only / informational-only</td><td>Three of the four variants report <code>rawWaste</code> in <code>mbSeconds</code>: <code>variant: &#39;wasteModel&#39;</code> passes through its own <code>wastedMBSeconds</code>; <code>&#39;idleCores&#39;</code> uses <code>idleRateFraction × allocatedMBSeconds</code>, with <code>idleRateFraction</code> measured against the same allocated core time as <code>utilization</code> and <code>allocatedMBSeconds</code> the run&#39;s allocated memory-seconds from <code>computeAllocation()</code> (executor heap plus overhead over each executor&#39;s alive time; Spark&#39;s defaults when the memory properties are not logged); <code>&#39;memoryBand&#39;</code> with <code>rule: &#39;heapOverProvisioned&#39;</code> uses <code>(allocatedBytes − heap) in MB × executorSeconds</code>, the seconds the run&#39;s executors were alive. The <code>dataUnavailable</code> shape has no inputs at all: informational-only</td></tr><tr><td><code>utilization</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>(1 − utilizationFraction) × allocatedCoreMs / 3.6e6</code>, where <code>allocatedCoreMs</code> is cores × time alive over every executor (<code>allocatedCoreMs()</code>, the figure behind <code>metrics.allocation.coreHours</code>), so it never exceeds the allocation. The same figure in core-milliseconds is <code>idleCoreTimeMs</code></td></tr><tr><td><code>coreLocality</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreMs</code>: <code>nonLocalTaskCount × NETWORK_FETCH_PENALTY_MS</code></td></tr><tr><td><code>autoscalingChurn</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>shortLivedExecutorCount × EXECUTOR_STARTUP_OVERHEAD_MS / 3.6e6</code></td></tr><tr><td><code>pythonUdf</code></td><td>sql</td><td>informational-only</td><td>no waste formula: the gain from Arrow-optimized or pandas UDFs depends on how much of the stage is the UDF body, which the log does not record</td></tr><tr><td><code>configAudit</code></td><td>config</td><td>informational-only</td><td>a config-drift check standing alone; no waste formula</td></tr><tr><td><code>jobFailureRate</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>failedJobCount × avgJobDurationMs / 3.6e6</code></td></tr><tr><td><code>cachingOpportunity</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>bytes</code> with <code>reread: true</code>: <code>totalReadBytes</code>, the input the queries sharing the relation scanned, measured and not a time. <code>assumptions</code> names how many queries it counts; a cache would still read the data once</td></tr><tr><td><code>cacheUtilization</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>ms</code> (printed as core time): uncached-or-spilled bytes <code>/ RE_READ_THROUGHPUT_BPS</code>, summed as task time and not divided over executors, where the never-cached partitions&#39; bytes are extrapolated from the cached partitions&#39; own average size (<code>memorySize + diskSize</code>, over <code>numCachedPartitions</code>), plus <code>diskSize</code> again for the already-cached-but-on-disk partitions&#39; own re-read cost. The <code>storageUnobserved</code> caveat (<code>dataUnavailable</code>) has no sizes: informational-only</td></tr><tr><td><code>stageFailed</code></td><td>stage</td><td>informational-only</td><td>no waste formula</td></tr><tr><td><code>failures</code></td><td>stage</td><td>informational-only</td><td>no waste formula</td></tr><tr><td><code>incompleteRun</code></td><td>app</td><td>informational-only</td><td>no waste formula</td></tr></tbody></table>',17)])])}const g=t(s,[["render",c]]);export{m as __pageData,g as default};

package/public/docs/assets/{contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js → contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.lean.js}

@@ -1 +1 @@
-import{_ as t,o,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Caveats, tuning and coverage","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md","filePath":"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md"}'),s={name:"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md"};function c(r,e,i,n,l,h){return o(),d("div",null,[...e[0]||(e[0]=[a("",13)])])}const p=t(s,[["render",c]]);export{g as __pageData,p as default};
+import{_ as t,o,c as a,a5 as d}from"./chunks/framework.DSg0KOwT.js";const m=JSON.parse('{"title":"Caveats, tuning and coverage","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md","filePath":"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md"}'),s={name:"contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.md"};function c(r,e,i,n,l,h){return o(),a("div",null,[...e[0]||(e[0]=[d("",17)])])}const g=t(s,[["render",c]]);export{m as __pageData,g as default};