sparkforensics-server 0.5.1 → 0.6.0

package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.BEepbDDL.js

@@ -0,0 +1,24 @@
+import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"CI and automation","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/ci-and-automation.md","filePath":"user-guide/getting-started/ci-and-automation.md"}'),i={name:"user-guide/getting-started/ci-and-automation.md"};function d(n,e,c,r,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s(`<h1 id="ci-and-automation" tabindex="-1">CI and automation <a class="header-anchor" href="#ci-and-automation" aria-label="Permalink to &quot;CI and automation&quot;">​</a></h1><p>How to run SparkForensics from a pipeline or script with the <code>analyze</code> CLI: its output formats, the extra data in the JSON report, and gating on several budgets and candidates.</p><p>In a pipeline or a script, use the <code>analyze</code> CLI instead of the browser dashboard. It parses the same event logs and exits non-zero when a run crosses a threshold you set, so it can gate a build:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -p</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-cli</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-analyze</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt; [--max-runtime </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ms]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-skew </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ratio]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  [--max-spill </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">gb]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-failed-task-rate </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--min-efficiency </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  [--out </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">path]</span></span></code></pre></div><p>Output is JSON by default; <code>--format md</code> writes the Markdown report instead. Budget violations and inconclusive budgets print to stderr as <code>[violation] ...</code> and <code>[inconclusive] ...</code> lines. The exit code is 0 when every budget passes, 1 when one is violated, 3 when none is violated but one is inconclusive, 2 for a usage error (bad flags or arguments, an invalid <code>--thresholds</code> or <code>--budgets</code> file), 4 when the candidate log can&#39;t be read or parsed or its History Server fetch fails, 5 when the <code>--baseline</code> log can&#39;t be read or parsed, and 6 for an internal error, including a failed <code>--export-html</code>. When several of these apply, the worst wins in this order: 6, 5, 4, 1, 3, 0. A usage error exits 2 before any log is read.</p><p><code>--min-efficiency</code> checks busy core time, the share of executor core time that ran tasks (100 minus the dashboard&#39;s Unused core time). It is not the dashboard&#39;s Efficiency tile, which is the share of wall-clock with a stage running, so the two can differ widely on the same run.</p><p>The command ships in the <code>sparkforensics-cli</code> package. To install it once: <code>npm i -g sparkforensics-cli</code>, then run <code>sparkforensics-analyze</code> directly. <code>npx sparkforensics-analyze &lt;file&gt;</code> is a shorter way to run the same command.</p><p>Point it at a single event-log file, or at one run&#39;s <code>eventlog_v2_*</code> rolling-log directory; any other directory is rejected. Output goes to stdout; pass <code>--out &lt;path&gt;</code> to write it to that file instead.</p><p>Pass <code>--export-html &lt;dir&gt;</code> to write a self-contained HTML dashboard for the run into <code>&lt;dir&gt;</code> (which must not already exist or must be empty). Open <code>&lt;dir&gt;/index.html</code> directly in a browser over <code>file://</code>, with no server, to get the same interactive dashboard offline, without the docs links. This makes it easy to archive or share a run. <code>--redact</code> applies to the exported report too.</p><h2 id="machine-readable-fixes-and-costs" tabindex="-1">Machine-readable fixes and costs <a class="header-anchor" href="#machine-readable-fixes-and-costs" aria-label="Permalink to &quot;Machine-readable fixes and costs&quot;">​</a></h2><p>Each row of the JSON report&#39;s <code>findings</code> array carries two fields for a script or tuning loop that acts on the output without reading prose.</p><p><code>remediation</code> is an array of the changes the row&#39;s <code>recommendation</code> names, in structured form. It is empty when the recommendation names no Spark property and no change to the job.</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;remediation&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;kind&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;conf&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;key&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;spark.sql.shuffle.partitions&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;direction&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;increase&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;suggested&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">800</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span></code></pre></div><ul><li><code>kind</code> is <code>&quot;conf&quot;</code> (a Spark property) or <code>&quot;code&quot;</code> (a change to the job&#39;s code or data). A consumer that reads <code>key</code> checks <code>kind</code> first.</li><li><code>direction</code> is <code>&quot;increase&quot;</code> or <code>&quot;decrease&quot;</code> (move the current value that way) or <code>&quot;set&quot;</code> (take the <code>suggested</code> value, for a switch or a class name).</li><li>Entries apply together unless the <code>recommendation</code> words them as alternatives (&quot;either ... or&quot;). An idle-capacity finding (<code>utilization</code>, <code>memoryUtilization</code> with <code>variant: &quot;idleCores&quot;</code>) does this with dynamic allocation off or unset: <code>set spark.dynamicAllocation.enabled</code> and <code>decrease spark.executor.instances</code> are alternatives, so apply one. With dynamic allocation on, it lowers <code>spark.dynamicAllocation.maxExecutors</code>, and also <code>spark.dynamicAllocation.minExecutors</code> when the run logs a value above 0.</li><li><code>suggested</code> is the value the detector computed, or <code>null</code> when it computes none. Counts are numbers, switches are booleans, sizes carry a Spark unit suffix (<code>&quot;1024m&quot;</code>).</li></ul><p>A <code>code</code> entry has no <code>key</code>, <code>direction</code> or <code>suggested</code>, only a <code>hint</code>:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;remediation&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;kind&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;code&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;hint&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;salt the key or repartition on a better key&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }]</span></span></code></pre></div><p>Skew and straggler findings carry one when no property can fix them: the stage&#39;s <code>evidence.origin</code> is <code>other</code>, or AQE skew-join handling is already on. The <code>hint</code> is the remedy the <code>recommendation</code> gives.</p><p><code>impactEstimate.coreTimeMs</code> is the busy core time the fix removes: the executor task time, in core-milliseconds, next to the <code>wallClock</code> range (elapsed time). It is a <code>{ &quot;low&quot;: ..., &quot;high&quot;: ... }</code> range with <code>low</code> equal to <code>high</code>, or <code>null</code> when the detector measures no such figure: <code>null</code> means unknown, never zero. Only figures read from the log count: GC time, the executor time of retried and discarded speculative attempts, and for skew and straggler findings the task time the fix removes from the slow tasks. These are set even when the log has no executor core data. A finding with only a wall-clock claim has <code>coreTimeMs: null</code>: its elapsed time is not converted to core time. Task run time is used, not <code>executorCpuTime</code>, so Python worker CPU is not missed.</p><p>Findings whose only figure is bytes or memory-time have <code>coreTimeMs: null</code>, as do findings whose core figure rests on an assumed constant: <code>coreLocality</code>&#39;s per-task fetch penalty and the executor-hours or job-hours of <code>autoscalingChurn</code> and <code>jobFailureRate</code>. Their <code>rawWaste</code> is unchanged.</p><p><code>coreTimeMs</code> never includes idle capacity. A finding whose waste is allocated cores that ran no task (<code>utilization</code>, <code>stageShape</code>&#39;s low parallelism and task/stage skew rows) has <code>coreTimeMs: null</code> and keeps its figure in <code>impactEstimate.rawWaste</code> with <code>&quot;idle&quot;: true</code>; every surface labels it &quot;of idle core capacity&quot;, not &quot;of core time&quot;. A stage&#39;s slow tail is counted once: when <code>skew</code> and <code>straggler</code> both flag the same stage, <code>skew</code> carries the removed task time and <code>straggler</code> has <code>null</code>.</p><p><code>impactEstimate.idleCoreTimeMs</code> is the idle capacity as a number a script can rank by: the <code>utilization</code> finding&#39;s allocated core time that ran no task, in core-milliseconds, as a <code>{ &quot;low&quot;: ..., &quot;high&quot;: ... }</code> range with <code>low</code> equal to <code>high</code>. It is the run&#39;s allocated core-milliseconds (<code>allocation.coreHours</code>) minus the busy core-milliseconds, never below zero and never above the allocation. It is the same quantity as the finding&#39;s <code>rawWaste</code>, kept apart from <code>coreTimeMs</code>, which is busy time a fix removes. The field is absent on every other finding, and on a <code>utilization</code> finding the log gives no executor cores to allocate. The <code>memoryUtilization</code> finding with <code>variant: &quot;idleCores&quot;</code> is the memory view of the same condition and carries no <code>idleCoreTimeMs</code>.</p><p>A finding on a serial basis whose figure is modeled has no core time either. <code>partitionSizing</code>&#39;s <code>lowShuffleParallelism</code> reports <code>rawWaste</code> in <code>ms</code>: stage wall-clock that more partitions shorten, not task work they remove. It has <code>coreTimeMs: null</code> and is ranked on <code>wallClock</code>, which models the stage with its longest task split evenly across the partition count the stage needs.</p><p>When the effective <code>spark.sql.shuffle.partitions</code> (logged, else Spark&#39;s 200) is already at or above the count a low-parallelism shuffle stage needs, the property is not what limits that stage: the recommendation points at the stage&#39;s own partitioning (<code>repartition(n)</code> or RDD parallelism) and <code>remediation</code> is empty. When AQE coalescing is on and the stage ran fewer tasks than the property, AQE merged the partitions and the recommendation points at <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code> instead. <code>shuffle</code> and <code>partitionSizing</code> (<code>lowShuffleParallelism</code>) findings carry <code>evidence.partitions</code> (<code>raise</code>, <code>sufficient</code>, <code>aqeCoalesced</code> or <code>ownPartitioning</code>) for the case.</p><p>A <code>remediation</code> that sets a property to a fixed value (for example <code>spark.sql.adaptive.skewJoin.enabled</code>, <code>spark.speculation</code> or <code>spark.dynamicAllocation.enabled</code>) is left out when the run&#39;s effective conf already has that value. The effective value is the logged property, else Spark&#39;s default for the run&#39;s <code>sparkVersion</code>; the defaults modeled are <code>spark.sql.adaptive.enabled</code> (off before Spark 3.2, on from 3.2), <code>spark.sql.adaptive.skewJoin.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> (both on from 3.0), and, for every version, <code>spark.sql.shuffle.partitions</code> (200) and <code>spark.sql.autoBroadcastJoinThreshold</code> (10 MiB). Spark before 3.0 has no AQE skew-join handling, so a skew finding on such a run suggests no conf. Booleans compare case-insensitively. The recommendation then stops naming that property and points at the remedy left (for example &quot;AQE skew-join handling is already on, so salt the key or repartition on a better key&quot;), so the text and <code>remediation</code> never disagree. The dashboard&#39;s one-line fix for a group of such findings follows the same effective conf.</p><p>Two effective settings change which fix is offered. Skew-join handling counts as already on only when <code>spark.sql.adaptive.enabled</code> is not effectively <code>false</code>; with AQE off (logged, or the default on Spark 3.0 and 3.1), the skew findings suggest setting <code>spark.sql.adaptive.enabled</code> to <code>true</code> instead. When <code>spark.sql.autoBroadcastJoinThreshold</code> is logged <code>-1</code> (auto-broadcast disabled), an over-broadcast finding has an empty <code>remediation</code> and points at removing the <code>broadcast()</code> hint.</p><p>Skew-join handling is only suggested for a stage that reads a shuffle in a SQL execution whose plan has a sort-merge or shuffled-hash join. Skew findings carry <code>evidence.origin</code>: <code>shuffleJoin</code> (the conf above applies), <code>inputScan</code> (a stage reading uneven input files: the remediation lowers <code>spark.sql.files.maxPartitionBytes</code>) or <code>other</code> (no conf is suggested; the <code>remediation</code> holds a <code>code</code> entry). <code>shufflePartitionSkew</code> carries the same field but is judged on shuffle-read sizes, so it is only ever <code>shuffleJoin</code> or <code>other</code>. A <code>stageSlowness</code> finding carries <code>evidence.reads</code> (<code>shuffle</code>, <code>input</code> or <code>other</code>) and suggests shuffle partitions only for <code>shuffle</code>; <code>spill</code> and <code>tinyTask</code> carry the same <code>reads</code> key and gate their shuffle-partition advice the same way. <code>straggler</code> carries <code>evidence.origin</code> and the same skew advice as a skew finding on that stage. <code>coldStart</code> and <code>autoscalingChurn</code> carry <code>evidence.dynamicAllocation</code> (<code>on</code> or <code>off</code>) and suggest no dynamic-allocation property when it is <code>off</code>. <code>underBroadcast</code> and <code>overBroadcast</code> carry <code>evidence.broadcastThreshold</code> (<code>limits</code>, <code>notLimiting</code> or <code>disabled</code>) and suggest changing the threshold only when it limits the join.</p><p>The CLI also supports fetching a run directly from a reachable Spark History Server (<code>--shs-base-url</code>/<code>--app-id</code>/<code>--attempt-id</code>) instead of a local file, comparing a candidate run against a baseline with regression gating (<code>--baseline</code>/<code>--max-regression-pct</code>/<code>--regression-metric</code>/ <code>--fail-on-introduced</code>; the baseline is a local file or rolling-log directory, no History Server), several regression budgets and several candidates in one call (<code>--regression-budget</code>/<code>--budgets</code>, see <a href="#several-budgets-and-several-candidates">Several budgets and several candidates</a>), rewriting run-specific text before stages are paired (<code>--normalize-path</code>, see <a href="#pairing-stages-across-runs">Pairing stages across runs</a>), redacting the app id, the app name and any host/IP tokens before sharing output (<code>--redact</code>), and narrowing the findings to certain impact bands, types, or a stage (<code>--impact</code>/<code>--type</code>/<code>--stage</code>), and tuning detector thresholds from a file (<code>--thresholds</code>, below). Run it with <code>--help</code> for the full flag list.</p><p>Same caveat as above: if the History Server is only reachable through an SSH bastion, <code>--shs-base-url</code> can&#39;t reach it either: see <a href="./../alternative-log-retrieval/ssh-bastion.html#behind-an-ssh-bastion">Behind an SSH bastion</a>.</p><p>Running in Airflow instead of a plain CI pipeline? See <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a>, an Airflow operator that wraps the CLI and acts on the result after each Spark job, so you don&#39;t have to wire up the call yourself.</p><p>Want an AI assistant to diagnose a run directly, without the dashboard or a CI gate? See <a href="./../mcp-tools.html">MCP tools reference</a>.</p><h2 id="write-targets" tabindex="-1">Write targets <a class="header-anchor" href="#write-targets" aria-label="Permalink to &quot;Write targets {#write-targets}&quot;">​</a></h2><p>The JSON report has a top-level <code>writeTargets</code> object listing every SQL write in the run, so a script can check where a job wrote without reading the plan. It has no Markdown counterpart. The MCP <code>diagnose_run</code> tool returns it too.</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;writes&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;sqlExecutionId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">3</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;nodeId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;e3:n0&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;command&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;InsertIntoHadoopFsRelationCommand&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;recognized&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;kind&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;target&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;hdfs://nn/sandbox/out/t1&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;outputRows&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">5000000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;mergeRows&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;raw&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Execute InsertIntoHadoopFsRelationCommand hdfs://nn/sandbox/out/t1, false, Parquet, ...&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;executionsWithoutPlan&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;sqlExecutionId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">4</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;reason&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;noPlan&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;skippedLines&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><ul><li><p><code>writes</code> holds one row per write node, ordered by SQL execution id and then by position in the plan. A <code>DeltaTable</code> API merge has no write node: it is one extra row per merge, see <a href="#delta-writes">Delta writes</a>. <code>command</code> is the plan node&#39;s name without the <code>Execute </code> prefix or an <code>Exec</code> suffix, <code>nodeId</code> is the plan node&#39;s id, and <code>raw</code> is the node&#39;s full plan string as the log recorded it.</p></li><li><p><code>kind</code> says how to read <code>target</code>:</p><ul><li><code>path</code> is a filesystem location.</li><li><code>table</code> is a table name whose catalog the command states or implies: a Hive or V1 command&#39;s <code>database.table</code>, a DataSource V2 name with a catalog part (<code>catalog.namespace.table</code>), or a Delta command&#39;s table as <code>database.table</code> (a table in the session catalog) or <code>catalog.database.table</code>.</li><li><code>unqualifiedTable</code> is a DataSource V2 name with no catalog part, such as <code>default.events</code> or <code>events</code>. The log does not say which catalog it lives in, so a check on its database part alone can match the wrong catalog.</li><li><code>jdbcTable</code> is a <code>dbtable</code> of a JDBC write. It names a table in an external database, not in a Spark catalog, and the database itself (the JDBC url) is not reported.</li></ul><p>Every target is copied verbatim from the plan string: nothing is resolved, so a relative path, an unexpanded <code>\${var}</code> placeholder or a bare table name appears exactly as the log states it. Backticks around table name parts are dropped.</p></li><li><p><code>target</code> and <code>kind</code> are <code>null</code> when the target cannot be determined. That covers a node whose plan string has no target (Spark omits it for some commands, and Delta command nodes carry none on real logs, which <a href="#delta-writes">Delta writes</a> covers), a redacted option value, a plan string that names more than one candidate target, and a plan string Spark cut short. A target is reported only when its whole text is followed by a delimiter in the plan string: a path cut by <code>spark.sql.maxMetadataStringLength</code> or a name with a <code>...</code> marker in it is <code>null</code>, never a partial path. A trailing <code>... N more fields</code> after the target, such as the column list of <code>InsertIntoHadoopFsRelationCommand</code>, does not affect the target before it. Treat a <code>null</code> target as unknown, not as inside or outside any location.</p></li><li><p><code>outputRows</code> is the node&#39;s own <code>number of output rows</code> SQL metric. It is <code>null</code> when the log has no such metric for that node, which is the case for the DataSource V2 and Delta write nodes that report other metrics.</p></li><li><p><code>mergeRows</code> holds the row counts of a Delta MERGE: <code>inserted</code>, <code>updated</code>, <code>deleted</code> and <code>copied</code>, read from the <code>MergeIntoCommand</code> node&#39;s SQL metrics (<code>number of inserted rows</code>, <code>number of updated rows</code>, <code>number of deleted rows</code> and <code>number of target rows rewritten unmodified</code>; older Delta releases name the last one <code>number of rows copied</code>). It is <code>null</code> for every other command, for a MERGE node that reports none of the four, and for a <code>DeltaMerge</code> row (see <a href="#delta-writes">Delta writes</a>). To check that two runs merged the same rows, compare <code>inserted + updated + deleted</code>. <code>copied</code> counts target rows Delta rewrote unchanged, which depends on how the table&#39;s files are laid out.</p></li><li><p><code>recognized</code> is <code>true</code> for the commands below and <code>false</code> for a write-like node that is not in that list. An unrecognized write always has <code>target</code> and <code>kind</code> <code>null</code>, and <code>raw</code> is the only information about it.</p></li><li><p><code>executionsWithoutPlan</code> lists the SQL executions whose writes the report cannot see, each with a <code>reason</code>. <code>noPlan</code> means the execution started but the log has no plan for it, such as an execution still running when the log was cut off (a plan is read when its execution ends). <code>unreadableStart</code> means the execution&#39;s start event was skipped because it could not be read, for example a plan nested deeper than 500 nodes or a start line cut off at the end of the log. A write in one of these executions is not in <code>writes</code>, so a non-empty list means the report may be incomplete.</p></li><li><p><code>skippedLines</code> is the number of log lines the parser could not read. A line cut off before its execution id is not in <code>executionsWithoutPlan</code>, so a non-zero count means a write may be missing even when that list is empty. It is <code>null</code> when the count is unknown.</p></li></ul><p>Recognized commands and where their target comes from:</p><table tabindex="0"><thead><tr><th>Command</th><th>Target</th></tr></thead><tbody><tr><td><code>InsertIntoHadoopFsRelationCommand</code></td><td><code>path</code>: the first argument, including for an <code>INSERT OVERWRITE ... PARTITION</code> that prints a static-partition map after it</td></tr><tr><td><code>InsertIntoHiveTable</code>, <code>CreateDataSourceTableAsSelectCommand</code>, <code>CreateHiveTableAsSelectCommand</code>, <code>OptimizedCreateHiveTableAsSelectCommand</code></td><td><code>table</code>: the first argument</td></tr><tr><td><code>SaveIntoDataSourceCommand</code></td><td><code>path</code> from the <code>path</code> option, or <code>jdbcTable</code> from the JDBC <code>dbtable</code>/<code>table</code> option</td></tr><tr><td><code>AppendData</code>, <code>OverwriteByExpression</code>, <code>OverwritePartitionsDynamic</code>, <code>ReplaceData</code>, <code>WriteDelta</code>, <code>WriteToDataSourceV2</code>, <code>AppendDataExecV1</code>, <code>OverwriteByExpressionExecV1</code></td><td><code>table</code> or <code>unqualifiedTable</code>: the <code>table=</code> of the connector&#39;s write object, for example Iceberg&#39;s <code>IcebergWrite(table=..., ...)</code>. For a Delta table, see <a href="#delta-writes">Delta writes</a></td></tr><tr><td><code>CreateTableAsSelect</code>, <code>AtomicCreateTableAsSelect</code>, <code>ReplaceTableAsSelect</code>, <code>AtomicReplaceTableAsSelect</code></td><td><code>table</code> or <code>unqualifiedTable</code>: the identifier after the catalog object, which does not name the catalog</td></tr><tr><td><code>WriteIntoDelta</code>, <code>WriteIntoDeltaCommand</code>, <code>UpdateCommand</code>, <code>DeleteCommand</code>, <code>MergeIntoCommand</code></td><td><code>path</code>: the first argument when it is a <code>delta.</code> path table (never for <code>MergeIntoCommand</code>, whose plan string also prints the source); otherwise as in <a href="#delta-writes">Delta writes</a></td></tr><tr><td><code>CreateDeltaTableCommand</code>, <code>OptimizeTableCommand</code>, <code>RestoreTableCommand</code>, <code>DeltaReorgTableCommand</code></td><td><code>path</code>: the first argument when it is a <code>delta.</code> path table, otherwise <code>null</code></td></tr><tr><td><code>DeltaMerge</code></td><td>not a plan node: one row per <code>DeltaTable</code> API merge, see <a href="#delta-writes">Delta writes</a></td></tr></tbody></table><p>A plan node not in the table is a write when its operator name (the first word of the node name after any leading <code>Execute</code>, so not the relation or table a scan prints after it), split into CamelCase words, contains <code>Write</code>, <code>Insert</code>, <code>Save</code>, <code>Overwrite</code>, <code>Append</code>, <code>Merge</code>, <code>Update</code>, <code>Delete</code>, <code>Truncate</code>, <code>Replace</code>, <code>Drop</code>, <code>Load</code>, <code>Vacuum</code>, <code>Convert</code>, <code>Clone</code>, <code>Restore</code>, <code>Optimize</code>, <code>Alter</code>, <code>Reorg</code>, <code>Rename</code> or <code>Call</code>, or contains <code>TableAsSelect</code>, <code>AddPartition</code>, <code>DropPartition</code>, <code>RenamePartition</code> or <code>RecoverPartitions</code>. That covers table DDL and <code>ADD PARTITION ... LOCATION</code>, and an Iceberg procedure call (<code>Call</code>), whose target is a procedure argument and not a path, so it is reported with a <code>null</code> target. A name containing <code>Join</code> (<code>SortMergeJoin</code>) and these operators that share a word with a write but write nothing are not: <code>WriteFiles</code> (the child of a write command), <code>AppendColumns</code>, <code>AppendColumnsWithObject</code>, <code>MergeRows</code>, <code>StateStoreSave</code>, <code>StateStoreRestore</code>, <code>SessionWindowStateStoreSave</code>, <code>SessionWindowStateStoreRestore</code> and <code>UpdateEventTimeWatermarkColumn</code>. The classification leans toward reporting too much: a node that matches by name is listed with a <code>null</code> target rather than dropped.</p><h3 id="delta-writes" tabindex="-1">Delta writes <a class="header-anchor" href="#delta-writes" aria-label="Permalink to &quot;Delta writes {#delta-writes}&quot;">​</a></h3><p>On a real log a Delta command node (<code>MergeIntoCommand</code>, <code>UpdateCommand</code>, <code>DeleteCommand</code>, <code>WriteIntoDelta</code>, <code>SaveIntoDataSourceCommand</code>) is a bare name: its plan string has no target. The target is read from up to two other places, in this order, and is <code>null</code> when neither names exactly one target.</p><ol><li>The command&#39;s <code>Arguments:</code> line in Spark&#39;s physical plan description. The parser drops the rest of that text to save memory, but keeps this one line for these commands. A quoted table name that appears once on the line is the target, as <code>kind: &quot;table&quot;</code> (<code>spark_catalog</code> is dropped, so it reads <code>database.table</code>). A <code>SaveIntoDataSourceCommand</code> is read this way only when the line shows a Delta source, and its target is the <code>path</code> option. A description that spans two read chunks is not kept for a <code>SaveIntoDataSourceCommand</code>, which prints its whole query plan under it; its target is then <code>null</code>.</li><li>For <code>MergeIntoCommand</code>, <code>UpdateCommand</code> and <code>DeleteCommand</code> only, the <code>_delta_log</code> path in the plans of the executions that share the command&#39;s root execution, as <code>kind: &quot;path&quot;</code>. Exactly one distinct path must appear. It applies only when the command is its own root execution, and a source that is itself a Delta table adds a second path and leaves the target <code>null</code>. The write commands (<code>WriteIntoDelta</code>, <code>WriteIntoDeltaCommand</code>, <code>SaveIntoDataSourceCommand</code>) never take this step, because a write may read another Delta table as its source: with no table or <code>path</code> option on the arguments line, their target is <code>null</code>.</li></ol><p>An append or overwrite of an existing Delta table (<code>saveAsTable</code> in <code>append</code> or <code>overwrite</code> mode on an existing table, <code>INSERT INTO</code>, <code>INSERT OVERWRITE</code>) prints the table object as its first argument, and that names the table: <code>kind: &quot;table&quot;</code> as <code>database.table</code> (<code>catalog.database.table</code> outside the session catalog), or <code>kind: &quot;path&quot;</code> for a path-based table. A <code>CREATE TABLE ... AS SELECT</code>, <code>CREATE OR REPLACE TABLE</code> or <code>saveAsTable</code> that creates or replaces a table runs two executions under one root: the create or replace node, which names the table as <code>unqualifiedTable</code> because Delta&#39;s catalog object does not state its catalog name, and an <code>AppendDataExecV1</code> whose staged table has no name. That append takes the table its root&#39;s create or replace node names, and is <code>null</code> when the root has no single such node.</p><p>A merge made with <code>DeltaTable.merge(...).execute()</code> normally runs a <code>MergeIntoCommand</code> too (its description is <code>toDataset$ at DeltaMergeBuilder</code>), and is reported like a SQL MERGE, including two merges run from threads of one session, whose sub-queries are told apart by their root execution. Some logs have no such command execution for an API merge. Its queries are then separate root executions, and the only things tying them together are a <code>MERGE operation</code> description and adjacent execution ids. Each run of consecutive ids with such a description is reported as one write with the synthetic command <code>DeltaMerge</code>, <code>kind: &quot;path&quot;</code> and the table path from the <code>_delta_log</code> of the plans in the run. The log holds no table name for these merges. The run must hold a write phase (a description containing <code>writing</code> or <code>rewriting</code>), because a run that only scanned may have read the merge&#39;s source. The target is <code>null</code> when the run has no write phase, names zero or several paths, has a plan or start time missing, or has start times that overlap another run&#39;s (merges on concurrent threads); <code>raw</code> says which. Two merges whose executions have consecutive ids are one run and name two paths, so they get no target. A description a user sets that starts with <code>MERGE operation</code> is indistinguishable from Delta&#39;s and is reported the same way. <code>raw</code> reads <code>Delta MERGE operation, executions 10-17</code>.</p><p>Only SQL writes are covered. Writes made outside Spark SQL, such as an RDD <code>saveAsTextFile</code> or a direct filesystem call from the driver, do not appear in the event log and are not in <code>writeTargets</code>.</p><h2 id="generator" tabindex="-1">Generator <a class="header-anchor" href="#generator" aria-label="Permalink to &quot;Generator&quot;">​</a></h2><p>The CLI&#39;s JSON output carries a <code>generator</code> block naming the build that wrote it: <code>name</code> and <code>version</code> of the CLI package, and <code>buildId</code>, the build id of the analysis core the CLI loaded. Equal build ids mean equal analysis code. With <code>--baseline</code> the block is inside <code>candidate</code>. The MCP <code>diagnose_run</code> tool does not return it. The block holds no run identifiers, so <code>--redact</code> leaves it as is.</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;generator&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;sparkforensics-cli&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;version&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;0.7.0&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;buildId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;9c1f3a7e2b4d&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span></code></pre></div><h2 id="metrics-block" tabindex="-1">Metrics block <a class="header-anchor" href="#metrics-block" aria-label="Permalink to &quot;Metrics block&quot;">​</a></h2><p>The CLI&#39;s JSON output carries a <code>metrics</code> block next to the report, for scripts that act on a run without reading the report. With <code>--baseline</code> it is inside <code>candidate</code>. The MCP <code>diagnose_run</code> tool returns the same block. It has its own <code>schemaVersion</code>, separate from the report&#39;s. A figure the log cannot provide is <code>null</code>, never <code>0</code>: a log from a Spark version that records no CPU time has <code>executorCpuTimeMs: null</code>, and a log cut off before any executor joined has null allocation. The time, data and task figures, run-level and per row, include the work of every attempt of a resubmitted stage, failed attempts too, and the tasks of a failed attempt that end after it (killed or still running when it failed); skew describes the latest attempt.</p><table tabindex="0"><thead><tr><th>Field</th><th>Meaning</th></tr></thead><tbody><tr><td><code>runComplete</code></td><td><code>true</code> when the log has an application-end record. <code>false</code> means the log was cut off and every total covers only what it recorded.</td></tr><tr><td><code>time.wallClockMs</code></td><td>Application start to end; null without both.</td></tr><tr><td><code>time.executorCpuTimeMs</code></td><td>Summed CPU time of every task attempt, including failed attempts, retries and speculative copies that lost. Null when no task recorded any. Misses Python worker CPU, see <code>python</code>.</td></tr><tr><td><code>time.executorRunTimeMs</code>, <code>time.gcTimeMs</code></td><td>Summed run time and JVM GC time of every task attempt, counted like the CPU time.</td></tr><tr><td><code>data.memorySpillBytes</code>, <code>data.diskSpillBytes</code></td><td>Summed spill.</td></tr><tr><td><code>data.shuffleReadBytes</code>, <code>data.shuffleWriteBytes</code></td><td>Summed shuffle bytes, local plus remote on the read side.</td></tr><tr><td><code>data.inputBytes</code>, <code>data.outputBytes</code>, <code>data.outputRows</code></td><td>Summed input, output and rows written. <code>outputRows</code> is null when no task reported rows.</td></tr><tr><td><code>data.peakExecutionMemoryBytes</code></td><td>The largest per-task peak execution memory. Null when every task reports 0.</td></tr><tr><td><code>shape.taskCount</code>, <code>shape.stageCount</code></td><td>Distinct task records and stages.</td></tr><tr><td><code>shape.failedStageAttempts</code></td><td>Stage attempts that ended with a failure reason, including an attempt Spark then resubmitted (a fetch failure, for example).</td></tr><tr><td><code>shape.retriedStages</code></td><td>Stages submitted more than once.</td></tr><tr><td><code>shape.failedTasks</code>, <code>shape.retriedTasks</code></td><td>Task-level counts: tasks whose final attempt failed, and task attempts superseded by a retry.</td></tr><tr><td><code>shape.maxSkew</code></td><td>The largest stage skew ratio, the figure <code>--max-skew</code> checks (P95 over median, or max over median for a stage with few tasks; <code>skew.minTasksForP95</code> from <code>--thresholds</code> applies).</td></tr><tr><td><code>allocation.coreHours</code>, <code>allocation.memoryGbHours</code></td><td>See <a href="#allocation">Allocation</a>.</td></tr><tr><td><code>allocation.dynamicAllocation</code>, <code>allocation.executorsPeak</code>, <code>allocation.executorsMean</code>, <code>allocation.executorCores</code>, <code>allocation.executorSeconds</code></td><td>The executor allocation behind the hours, see <a href="#allocation">Allocation</a>.</td></tr><tr><td><code>python.shareOfTaskRunTime</code></td><td>See <a href="#python-share">Python share</a>.</td></tr><tr><td><code>stages</code></td><td>The per-stage rows.</td></tr></tbody></table><p><code>stages</code> holds one row per stage fingerprint, keyed by the same fingerprint the run comparison matches stages on (the normalized stage name plus the stage&#39;s plan nodes), so the same stage has the same key in a baseline and a candidate run. A stage repeated in a loop shares one key: its row sums the repeats and lists them in <code>stageIds</code>. A row carries <code>durationMs</code>, <code>executorCpuTimeMs</code>, <code>executorRunTimeMs</code>, <code>gcTimeMs</code>, the spill, shuffle, input and output figures, <code>outputRows</code>, <code>peakExecutionMemoryBytes</code>, <code>taskCount</code>, <code>failedTasks</code>, <code>retriedTasks</code> and <code>skew</code> (the largest skew ratio among the repeats) as above, plus <code>failed</code> (an attempt of the stage failed), <code>retried</code> (the stage was submitted more than once) and <code>python</code> flags. The stage-attempt figures, run-level and per row, are null for a stage record that carries no attempt count. With <code>--redact</code> the <code>metrics</code> and <code>effectiveConf</code> blocks are built from the redacted run the report uses, so they carry the same <code>host-N</code> and <code>app-N</code> pseudonyms: the keys come from redacted stage names.</p><h3 id="allocation" tabindex="-1">Allocation <a class="header-anchor" href="#allocation" aria-label="Permalink to &quot;Allocation&quot;">​</a></h3><p><code>allocation.coreHours</code> is the sum over executors of cores times hours alive. <code>allocation.memoryGbHours</code> is the sum of memory times hours alive, in GiB (1024 MiB). An executor is alive from its executor-added event to its first later executor-removed event. One with no removal event closes at the application end when the log has one. In a cut-off log (<code>runComplete</code> false) it closes at the last timestamp the log records (the latest application, stage or executor event), so the figure is a lower bound.</p><p>Cores per executor are the executor-added event&#39;s total cores. Under dynamic allocation or YARN defaults, where <code>spark.executor.cores</code> is not set, that is the core count the cluster manager actually granted; when the event carries none, <code>spark.executor.cores</code> is used. If neither exists the core-hours are null.</p><p>Memory per executor is the container size Spark requests: <code>spark.executor.memory</code> (1g when unset), plus the overhead, plus <code>spark.memory.offHeap.size</code> when <code>spark.memory.offHeap.enabled</code> is <code>true</code>, plus <code>spark.executor.pyspark.memory</code>. The overhead is <code>spark.executor.memoryOverhead</code>, else the legacy <code>spark.yarn.executor.memoryOverhead</code>, else the larger of 384 MiB and <code>spark.executor.memoryOverheadFactor</code> (default 0.1) times the executor memory. Memory GB-hours are null when the log records no Spark properties at all, or a memory property cannot be read. Driver resources are not included.</p><p>The same alive intervals give the executor figures, each null when the log has no executor-added event:</p><ul><li><code>allocation.executorSeconds</code> is the sum of seconds alive over executors.</li><li><code>allocation.executorsPeak</code> is the largest number of executors alive at once.</li><li><code>allocation.executorsMean</code> is <code>executorSeconds</code> divided by the seconds from application start to close (the end, or the last recorded timestamp in a cut-off log), so the mean times the run&#39;s wall clock is <code>executorSeconds</code>. It is null when the log has no application start.</li><li><code>allocation.executorCores</code> is the cores per executor when every executor has the same count, null when a count is unknown or they differ.</li></ul><p><code>allocation.dynamicAllocation</code> is <code>on</code> or <code>off</code> when the log records <code>spark.dynamicAllocation.enabled</code>, and <code>null</code> when it does not. Spark&#39;s default is off, but a key the log never recorded is not reported as off. It is read whether or not the log has executor events. The <code>coldStart</code> and <code>autoscalingChurn</code> findings&#39; <code>evidence.dynamicAllocation</code> reports <code>on</code> for any run not logged as off.</p><h3 id="python-share" tabindex="-1">Python share <a class="header-anchor" href="#python-share" aria-label="Permalink to &quot;Python share&quot;">​</a></h3><p><code>python.shareOfTaskRunTime</code> is the summed task run time of Python stages divided by the summed task run time of all stages; null when no task run time was recorded. It is <code>0</code> for a run with task time and no Python stage. Executor CPU time counts only the JVM task thread, so on a run with a high share it misses the CPU of the Python worker processes; treat <code>executorCpuTimeMs</code> as unreliable there.</p><p>A stage is a Python stage when either signal holds:</p><ul><li>a plan node attributed to it is a Python operator (<code>PythonRDD</code>, <code>BatchEvalPython</code>, <code>ArrowEvalPython</code>, <code>PythonMapInArrow</code>, or a pandas/Arrow grouped or map operator such as <code>FlatMapGroupsInPandas</code>, including suffixed variants such as <code>BatchEvalPythonUDTF</code> and <code>FlatMapGroupsInPandasWithState</code> and the Spark 4.1 <code>ArrowAggregatePython</code> and <code>ArrowWindowPython</code>), which catches Python UDFs and UDTFs inside SQL;</li><li>its name or call site names <code>PythonRDD</code> or <code>org.apache.spark.api.python</code>, which catches RDD lambdas that have no plan and stages that cannot be matched to one.</li></ul><p>The &quot;tasks mostly idle&quot; check that keeps a low-CPU stage from being treated as waiting on an external system uses the same test.</p><h3 id="other-surfaces" tabindex="-1">Other surfaces <a class="header-anchor" href="#other-surfaces" aria-label="Permalink to &quot;Other surfaces&quot;">​</a></h3><p>Every figure the metrics block shares with another surface reads the same core code as that surface. The run comparison (the dashboard&#39;s comparison view, the CLI&#39;s <code>--baseline</code> and the MCP <code>compare_runs</code> tool) uses the same sums for run time, CPU time, GC time, spill, input, output and task count, so a comparison metric equals the matching <code>metrics</code> field for the same run, including the work of failed and speculative task attempts. Per-stage views in the dashboard (the stage table and stage detail) describe the stage&#39;s latest attempt and the task attempts that won, so a stage&#39;s own figures can be smaller than its row in <code>metrics.stages</code>. The dashboard&#39;s Unused core time, the <code>utilization</code> finding&#39;s percentage and its idle figure all measure against available capacity, which is <code>allocation.coreHours</code> (cores times the hours each executor was alive), so under dynamic allocation a run is never idle for cores it did not hold. The utilization finding&#39;s <code>cpuUtilizationPct</code> is a share of that capacity, and is null, as <code>time.executorCpuTimeMs</code> is, when the log recorded no CPU time.</p><h2 id="effective-conf" tabindex="-1">Effective conf <a class="header-anchor" href="#effective-conf" aria-label="Permalink to &quot;Effective conf&quot;">​</a></h2><p>The JSON output also carries <code>effectiveConf</code> (with its own <code>schemaVersion</code>), the Spark properties the run started with, so a script can check that a <code>--conf</code> overlay took effect. It is null when the log records no Spark properties. With <code>--baseline</code> it is inside <code>candidate</code>. The MCP <code>diagnose_run</code> tool returns the same block.</p><ul><li><code>values</code>: property to value, for every property that is not withheld.</li><li><code>maskedKeys</code>: properties present in the log whose value is withheld.</li><li><code>absentKeys</code>: with <code>--conf-keys</code>, the requested properties the log does not contain.</li></ul><p>A value is withheld when its key or value matches Spark&#39;s default secret pattern (<code>(?i)secret|password|token|access[.]?key</code>), as Spark&#39;s own redaction does, when its key names another credential form (<code>passwd</code>, <code>pwd</code>, <code>pass</code>, <code>apiKey</code>, <code>accountKey</code> as in <code>fs.azure.account.key.*</code>, <code>privateKey</code>, <code>sas</code>, <code>sig</code>, <code>credential</code>), when the key or value matches the job&#39;s own <code>spark.redaction.regex</code> (when the log records one; if that pattern uses syntax JavaScript cannot evaluate, every value is withheld), or when the key or value matches <code>--conf-redact-regex &lt;pattern&gt;</code>. A withheld property shows that it is present, not what it is set to, so the output cannot tell you whether a masked property has the value you expected. No hash or other derivative of a withheld value is emitted.</p><p>In every other value, credentials are replaced with <code>[redacted]</code>: in a value that looks like a URL, <code>user:password@host</code> userinfo and the Oracle thin <code>user/password@host</code> form; in any value, <code>name=value</code> parameters named <code>password</code>, <code>passwd</code>, <code>pwd</code>, <code>pass</code>, <code>apikey</code>, <code>accountkey</code>, <code>sas</code>, <code>sig</code>, <code>signature</code> and similar (JDBC and ODBC strings, query strings, JVM options), including signature parameters such as Azure SAS <code>sig=</code> and <code>X-Amz-Signature=</code>. <code>--conf-keys a,b</code> narrows <code>values</code> and <code>maskedKeys</code> to the named properties.</p><p>With <code>--redact</code>, the host names the report pseudonymizes become <code>host-N</code> in every value: the values of <code>*.host</code> and <code>*.hostname</code> properties, executor hosts, and IP or EC2-style addresses. <code>spark.app.name</code> becomes the application&#39;s <code>app-N</code> pseudonym. A host name that appears only in some other property, such as <code>spark.yarn.historyServer.address</code>, is left as is; withhold it with <code>--conf-redact-regex</code> or leave it out with <code>--conf-keys</code>.</p><h2 id="pairing-stages-across-runs" tabindex="-1">Pairing stages across runs <a class="header-anchor" href="#pairing-stages-across-runs" aria-label="Permalink to &quot;Pairing stages across runs {#pairing-stages-across-runs}&quot;">​</a></h2><p>With <code>--baseline</code>, the <code>comparison</code> object pairs the stages of the two runs (<code>stagePairs</code>, with each pair&#39;s <code>deltas</code>) and reports <code>runtimeCoverage</code>, the share of both runs&#39; executor run time that sits in paired or replanned stages. <code>confidence</code> is <code>ok</code>, <code>low</code> or <code>insufficient</code>: <code>low</code> when the application names differ or <code>runtimeCoverage</code> is under 0.9, <code>insufficient</code> when neither run recorded any executor run time. <code>confidence</code> does not change an exit code. The fields and the matching rules are in <a href="./../mcp-tools/compare-runs.html"><code>compare_runs</code></a> and <a href="./../run-comparison/how-stages-are-matched.html">How stages are matched</a>.</p><p>When a stage writes to a path that differs per run, for example <code>{sandbox_root}/{session}/{variant}/{batch}</code> where the variant is <code>baseline</code> in one run and <code>c07b</code> in the other, the write stage cannot pair, and it is often the heaviest. Pass <code>--normalize-path &lt;regex&gt;</code>, repeatable, to rewrite what the patterns match, in every plan node&#39;s text, before stages are paired:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">sparkforensics-analyze</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> candidate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --baseline</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> baseline</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --normalize-path</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;/sandbox/[^/]+/[^/]+/&#39;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --format</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> json</span></span></code></pre></div><p>Every match is replaced with a fixed token. Despite the name it is a generic regular expression, not only for paths. It needs <code>--baseline</code> and applies to every candidate of a several-candidates run. It changes which stages pair, and so <code>stagePairs</code>, <code>runtimeCoverage</code> and <code>confidence</code>; findings and the other metrics are untouched. At most 16 patterns of 200 characters each. An invalid pattern, or one that matches the empty string, exits <code>2</code> before any log is read.</p><h2 id="several-budgets-and-several-candidates" tabindex="-1">Several budgets and several candidates <a class="header-anchor" href="#several-budgets-and-several-candidates" aria-label="Permalink to &quot;Several budgets and several candidates&quot;">​</a></h2><h3 id="several-regression-budgets" tabindex="-1">Several regression budgets <a class="header-anchor" href="#several-regression-budgets" aria-label="Permalink to &quot;Several regression budgets&quot;">​</a></h3><p><code>--max-regression-pct</code> checks one metric. To gate on several in one call, repeat <code>--regression-budget &lt;metric&gt;:&lt;pct&gt;</code>, or list them in a file passed with <code>--budgets &lt;file&gt;</code>. Both need <code>--baseline</code>, and each budget takes a <a href="./regression-metric-keys.html#regression-metric-keys">metric key</a> and a percentage of zero or more:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">sparkforensics-analyze</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> candidate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --baseline</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> baseline</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --regression-budget</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> wallClock:10</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --regression-budget</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> gcTime:25</span></span></code></pre></div><p>The <code>--budgets</code> file is JSON with one key, <code>regression</code>, mapping a metric key to its percentage:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;regression&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;wallClock&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">10</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;gcTime&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">25</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;failedTaskRate&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } }</span></span></code></pre></div><p>The CLI refuses to run, with exit code 2 and a message naming the problem, for an unknown top-level key, an unknown metric key, a percentage that is not a non-negative number (in the file a JSON number, not a string; on the flag plain digits such as <code>10</code> or <code>2.5</code>), or a file that can&#39;t be read or isn&#39;t valid JSON.</p><p>The budgets combine like this:</p><ul><li><code>--max-regression-pct</code> with <code>--regression-metric</code> (default <code>wallClock</code>) still works and counts as one more budget next to the new ones.</li><li>Each metric can be budgeted once across the legacy pair, the repeated flag and the file. A metric named twice is a usage error (exit 2), even when both give the same percentage.</li><li>Each budget is checked on its own and gives one <code>max-regression</code> result that carries <code>metric</code>, the key it checks. Any violated budget makes the run exit <code>1</code>; otherwise an inconclusive one makes it exit <code>3</code>. A budget on a metric the baseline or the candidate log can&#39;t provide is inconclusive, never a pass.</li></ul><h3 id="per-stage-regression-budgets" tabindex="-1">Per-stage regression budgets <a class="header-anchor" href="#per-stage-regression-budgets" aria-label="Permalink to &quot;Per-stage regression budgets&quot;">​</a></h3><p><code>--max-regression-pct</code> and <code>--regression-budget</code> judge a whole-run metric. To gate on the stages themselves, repeat <code>--stage-regression-budget &lt;metric&gt;:&lt;pct&gt;</code> (with <code>--baseline</code>). The budget fails when any paired stage&#39;s metric grew by more than the percentage against its baseline stage. The metric is one of <code>executorRunTime</code>, <code>executorCpuTime</code>, <code>memoryBytesSpilled</code>, <code>diskBytesSpilled</code>, <code>shuffleReadBytes</code> and <code>shuffleWriteBytes</code>:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">sparkforensics-analyze</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> candidate.zstd</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --baseline</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> baseline.zstd</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --stage-regression-budget</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> executorRunTime:25</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --stage-regression-budget</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> diskBytesSpilled:0</span></span></code></pre></div><ul><li>The stages come from the comparison&#39;s <code>stagePairs</code>. By default only <code>exact</code> and <code>structural</code> pairs count, since an <code>aligned</code> pair may compare different work. <code>--stage-quality exact,structural,aligned</code> chooses the qualities; each pair&#39;s <code>quality</code> and <code>score</code> are in the comparison output. Stages a re-plan left over are not paired and are not checked.</li><li>Each metric gives one <code>max-stage-regression</code> result carrying <code>metric</code>, whose detail names the worst offenders by <code>pairId</code>. A violation exits <code>1</code>.</li><li>A stage that grew from a zero baseline exceeds any percentage.</li><li>It is inconclusive (exit <code>3</code>) when no eligible stage pairs or the metric is missing on every one, never a pass. A comparison with <code>low</code> or <code>insufficient</code> confidence is still judged on the pairs it has.</li><li>A metric can be budgeted once. <code>--stage-quality</code> without <code>--stage-regression-budget</code>, an unknown metric or quality, and a metric named twice are usage errors (exit <code>2</code>), and so are <code>inputBytes</code> and <code>outputBytes</code>: workload volume has no regression direction.</li></ul><h3 id="several-candidates" tabindex="-1">Several candidates <a class="header-anchor" href="#several-candidates" aria-label="Permalink to &quot;Several candidates&quot;">​</a></h3><p>Pass two or more logs as positional arguments, with <code>--baseline</code>, to compare each against the same baseline. The baseline is parsed once. Output is NDJSON: one line per candidate, in argument order, written as each one finishes. <code>--out &lt;path&gt;</code> writes the lines to a file instead of stdout, and <code>--format ndjson</code> selects this output for a single candidate too. The mode can&#39;t be combined with <code>--export-html</code>, <code>--shs-base-url</code>, <code>--format json</code> or <code>--format md</code> (exit 2). <code>--redact</code>, <code>--thresholds</code>, <code>--impact</code>, <code>--type</code>, <code>--stage</code>, <code>--conf-keys</code>, <code>--conf-redact-regex</code> and every budget flag apply to each candidate.</p><p>Each line is one JSON object:</p><table tabindex="0"><thead><tr><th>Field</th><th>Meaning</th></tr></thead><tbody><tr><td><code>log</code></td><td>The candidate path, as given on the command line. With <code>--redact</code>, <code>candidate-&lt;n&gt;</code> instead, <code>n</code> being the candidate&#39;s 1-based position.</td></tr><tr><td><code>status</code></td><td><code>pass</code>, <code>violation</code>, <code>inconclusive</code> or <code>error</code>.</td></tr><tr><td><code>exitCode</code></td><td>The exit code this line alone would give: <code>0</code> for <code>pass</code>, <code>1</code> for <code>violation</code>, <code>3</code> for <code>inconclusive</code>, and for <code>error</code> <code>4</code> (the log can&#39;t be read or parsed) or <code>6</code> (an internal failure while analyzing it).</td></tr><tr><td><code>error</code></td><td>The message when <code>status</code> is <code>error</code>; otherwise <code>null</code>. With <code>--redact</code>, a generic message that names no path.</td></tr><tr><td><code>budgets</code></td><td>This candidate&#39;s budget results, each with <code>name</code>, <code>status</code> (<code>pass</code>, <code>violation</code> or <code>inconclusive</code>) and <code>detail</code>, plus <code>metric</code> on <code>max-regression</code> and <code>max-stage-regression</code>. Empty for an <code>error</code> line.</td></tr><tr><td><code>candidate</code></td><td>The candidate&#39;s report with its <code>metrics</code> and <code>effectiveConf</code> blocks, the same object the single-candidate JSON output carries under <code>candidate</code>. <code>null</code> for an <code>error</code> line.</td></tr><tr><td><code>comparison</code></td><td><code>verdict</code>, <code>confidence</code>, <code>reason</code>, <code>matchedCoverage</code>, <code>runtimeCoverage</code>, <code>metrics</code>, <code>findings</code>, <code>comparisonSchemaVersion</code>, <code>stagePairs</code>, <code>unmatched</code>, <code>replanned</code>, <code>bookkeepingStageIds</code> and <code>executionAlignment</code>, the same object the single-candidate JSON output carries under <code>comparison</code>. <code>null</code> for an <code>error</code> line.</td></tr></tbody></table><p>A line&#39;s <code>status</code> follows the single-candidate rules: <code>violation</code> if any budget result is a violation, else <code>inconclusive</code> if any is inconclusive (including the <code>run-complete</code> check for a log with no <code>ApplicationEnd</code>), else <code>pass</code>.</p><p>A candidate that can&#39;t be read or parsed doesn&#39;t stop the others. Its line has <code>status: &quot;error&quot;</code>, <code>exitCode: 4</code>, the message in <code>error</code>, and <code>null</code> for <code>candidate</code> and <code>comparison</code>, as a single-candidate run exits <code>4</code> for a candidate it can&#39;t parse. The process exit code is the worst line, in the order <code>6</code>, <code>5</code>, <code>4</code>, <code>1</code>, <code>3</code>, <code>0</code>, so an unreadable candidate outranks a violation and a violation outranks an inconclusive result.</p><p>Two failures stop the batch before any line is written:</p><ul><li>A usage error (bad flags or arguments, an invalid <code>--thresholds</code> or <code>--budgets</code> file) exits <code>2</code>.</li><li>A baseline that can&#39;t be read or parsed exits <code>5</code>. Fix the baseline and run the batch again.</li></ul><p>An internal failure that isn&#39;t tied to one candidate, such as an unwritable <code>--out</code> path, exits <code>6</code>.</p><p>With <code>--redact</code>, no candidate path is written anywhere, since event-log file names usually carry the app id. <code>log</code> and the <code>stderr</code> line prefixes name each candidate by its 1-based position (<code>candidate-1</code>, <code>candidate-2</code>, ...), and an <code>error</code> line carries a generic message instead of the parser&#39;s, which may quote the path: <code>Candidate 2 could not be read or parsed.</code>, or <code>could not be analyzed</code> for an internal failure. An unreadable baseline prints <code>The baseline could not be read or parsed.</code> to <code>stderr</code>. A single-candidate run does the same, naming <code>The baseline</code> or <code>The candidate</code>.</p>`,98)])])}const g=o(i,[["render",d]]);export{u as __pageData,g as default};

package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.BEepbDDL.lean.js

@@ -0,0 +1 @@
+import{_ as o,o as t,c as a,a5 as s}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"CI and automation","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/ci-and-automation.md","filePath":"user-guide/getting-started/ci-and-automation.md"}'),i={name:"user-guide/getting-started/ci-and-automation.md"};function d(n,e,c,r,l,h){return t(),a("div",null,[...e[0]||(e[0]=[s("",98)])])}const g=o(i,[["render",d]]);export{u as __pageData,g as default};

package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.CPowlb2q.js → user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js}

@@ -1,4 +1,4 @@
-import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"compare_runs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/compare-runs.md","filePath":"user-guide/mcp-tools/compare-runs.md"}'),n={name:"user-guide/mcp-tools/compare-runs.md"};function h(l,s,o,d,p,k){return a(),e("div",null,[...s[0]||(s[0]=[t(`<h1 id="compare-runs" tabindex="-1"><code>compare_runs</code> <a class="header-anchor" href="#compare-runs" aria-label="Permalink to &quot;\`compare_runs\`&quot;">​</a></h1><p>Reference for the <code>compare_runs</code> tool of the SparkForensics MCP server.</p><p>Compare two runs: the comparison page&#39;s verdict, metric deltas (<code>metrics</code>) and categorized findings delta (<code>findings</code>), and the share of executor run time in paired stages (<code>runtimeCoverage</code>). The result carries the same <code>verdict</code>, <code>confidence</code>, <code>reason</code>, <code>matchedCoverage</code>, <code>runtimeCoverage</code>, <code>metrics</code>, <code>findings</code>, <code>comparisonSchemaVersion</code>, <code>unmatched</code>, <code>replanned</code> and <code>bookkeepingStageIds</code> as the <code>comparison</code> object of the CLI&#39;s <code>--baseline</code> JSON output, plus <code>stagePairs</code> when <code>include</code> asks for it (the CLI always carries it), plus <code>runIdA</code> and <code>runIdB</code>. <code>metricDeltas</code> and <code>findingsDelta</code> repeat <code>metrics</code> and <code>findings</code> under their earlier names; they are deprecated and will be removed in a later release. <code>verdict</code> is the headline the dashboard&#39;s comparison page opens with (the baseline is <code>runIdA</code>/<code>sourceA</code>, the candidate <code>runIdB</code>/<code>sourceB</code>): a <code>title</code> such as &quot;The candidate finished 9.9s faster than the baseline (37%)&quot;, or a failed-job headline when either run had jobs fail, a <code>tone</code> (<code>better</code>, <code>worse</code>, <code>same</code> or <code>unknown</code>), and <code>sentences</code> naming which cost metrics and finding categories moved each way. When either log has no <code>ApplicationEnd</code> event, the title compares how much run time each log covers (&quot;The candidate&#39;s log covers 2.0s less run time than the baseline&#39;s&quot;) and the tone is <code>unknown</code>.</p><p>Parameters:</p><ul><li><code>runIdA</code> / <code>sourceA</code>: identify the baseline (same <code>source</code> shape as above)</li><li><code>runIdB</code> / <code>sourceB</code>: identify the candidate</li><li><code>redact</code>: <code>boolean</code> (default <code>false</code>), pseudonymizes any app id or host/IP tokens embedded in free text (stage names and similar) in the response: see the note at the top of this page</li><li><code>include</code>: <code>[&quot;stagePairs&quot;]</code> (optional), adds <code>stagePairs</code>, one row per paired stage. It is left out by default because it grows with the number of stages.</li><li><code>normalizePath</code>: <code>string[]</code> (optional), regular expressions whose every match in a plan node&#39;s text is replaced with a fixed token before stages are paired, so text that is specific to your runs does not keep the same stage from pairing. Despite the name it is a generic pattern, not only for paths. It affects stage pairing, <code>runtimeCoverage</code> and <code>confidence</code>, never the findings. At most 16 patterns of 200 characters each. An invalid pattern, or one that matches the empty string, fails the call before any log is read. A pattern that backtracks catastrophically can stall the server, and the length cap does not prevent that: an agent passing patterns should keep them simple and anchored.</li><li><code>format</code>: <code>&quot;json&quot; | &quot;md&quot;</code> (default <code>&quot;json&quot;</code>), switches <code>content[0].text</code> to a rendered Markdown report instead of JSON. <code>structuredContent</code> always stays JSON-shaped, regardless of <code>format</code>.</li></ul><p>Each side needs either a <code>runId</code> or a <code>source</code>, and you can mix them: a cached run ID for the baseline, a fresh file for the candidate.</p><p>Example call (add <code>&quot;include&quot;: [&quot;stagePairs&quot;]</code> to the arguments for the pair rows):</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"compare_runs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/compare-runs.md","filePath":"user-guide/mcp-tools/compare-runs.md"}'),n={name:"user-guide/mcp-tools/compare-runs.md"};function h(o,s,l,d,p,k){return a(),e("div",null,[...s[0]||(s[0]=[t(`<h1 id="compare-runs" tabindex="-1"><code>compare_runs</code> <a class="header-anchor" href="#compare-runs" aria-label="Permalink to &quot;\`compare_runs\`&quot;">​</a></h1><p>Reference for the <code>compare_runs</code> tool of the SparkForensics MCP server.</p><p>Compare two runs: the comparison page&#39;s verdict, metric deltas (<code>metrics</code>) and categorized findings delta (<code>findings</code>), and the share of executor run time in paired stages (<code>runtimeCoverage</code>). The result carries the same <code>verdict</code>, <code>confidence</code>, <code>reason</code>, <code>matchedCoverage</code>, <code>runtimeCoverage</code>, <code>metrics</code>, <code>findings</code>, <code>comparisonSchemaVersion</code>, <code>unmatched</code>, <code>replanned</code>, <code>bookkeepingStageIds</code> and <code>executionAlignment</code> as the <code>comparison</code> object of the CLI&#39;s <code>--baseline</code> JSON output, plus <code>stagePairs</code> when <code>include</code> asks for it (the CLI always carries it), plus <code>runIdA</code> and <code>runIdB</code>. <code>metricDeltas</code> and <code>findingsDelta</code> repeat <code>metrics</code> and <code>findings</code> under their earlier names; they are deprecated and will be removed in a later release. <code>verdict</code> is the headline the dashboard&#39;s comparison page opens with (the baseline is <code>runIdA</code>/<code>sourceA</code>, the candidate <code>runIdB</code>/<code>sourceB</code>): a <code>title</code> such as &quot;The candidate finished 9.9s faster than the baseline (37%)&quot;, or a failed-job headline when either run had jobs fail, a <code>tone</code> (<code>better</code>, <code>worse</code>, <code>same</code> or <code>unknown</code>), and <code>sentences</code> naming which cost metrics and finding categories moved each way. When either log has no <code>ApplicationEnd</code> event, the title compares how much run time each log covers (&quot;The candidate&#39;s log covers 2.0s less run time than the baseline&#39;s&quot;) and the tone is <code>unknown</code>.</p><p>Parameters:</p><ul><li><code>runIdA</code> / <code>sourceA</code>: identify the baseline (same <code>source</code> shape as above)</li><li><code>runIdB</code> / <code>sourceB</code>: identify the candidate</li><li><code>redact</code>: <code>boolean</code> (default <code>false</code>), pseudonymizes any app id or host/IP tokens embedded in free text (stage names and similar) in the response: see the note at the top of this page</li><li><code>include</code>: <code>[&quot;stagePairs&quot;]</code> (optional), adds <code>stagePairs</code>, one row per paired stage. It is left out by default because it grows with the number of stages.</li><li><code>normalizePath</code>: <code>string[]</code> (optional), regular expressions whose every match in a plan node&#39;s text is replaced with a fixed token before stages are paired, so text that is specific to your runs does not keep the same stage from pairing. Despite the name it is a generic pattern, not only for paths. It affects stage pairing, <code>runtimeCoverage</code> and <code>confidence</code>, never the findings. At most 16 patterns of 200 characters each. An invalid pattern, or one that matches the empty string, fails the call before any log is read. A pattern that backtracks catastrophically can stall the server, and the length cap does not prevent that: an agent passing patterns should keep them simple and anchored.</li><li><code>format</code>: <code>&quot;json&quot; | &quot;md&quot;</code> (default <code>&quot;json&quot;</code>), switches <code>content[0].text</code> to a rendered Markdown report instead of JSON. <code>structuredContent</code> always stays JSON-shaped, regardless of <code>format</code>.</li></ul><p>Each side needs either a <code>runId</code> or a <code>source</code>, and you can mix them: a cached run ID for the baseline, a fresh file for the candidate.</p><p>Example call (add <code>&quot;include&quot;: [&quot;stagePairs&quot;]</code> to the arguments for the pair rows):</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;compare_runs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;sourceA&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;baseline.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
@@ -41,6 +41,7 @@ import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;unmatched&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;replanned&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [],</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;bookkeepingStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;executionAlignment&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseExecutions&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candExecutions&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;pairedExecutions&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;bounded&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;agreement&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;accepted&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;metricDeltas&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;key&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;wallClock&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
@@ -52,4 +53,4 @@ import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;findingsDelta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;introduced&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;resolved&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] }</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each <code>findings.introduced</code>/<code>resolved</code> row is <code>{ rule, type, impactBand, baseCount, candCount, delta, stages }</code>, with <code>stages</code> naming the affected stages. A <code>metrics</code> row&#39;s <code>direction</code> is <code>improvement</code>, <code>regression</code>, <code>unchanged</code>, <code>neutral</code> (a volume metric, where more isn&#39;t worse) or <code>unavailable</code>, and an unavailable row carries an <code>unavailableReason</code>.</p><p>The comparison block (<code>comparisonSchemaVersion</code> <code>1</code>) holds:</p><ul><li><code>stagePairs</code> (only with <code>include</code>): one entry per pair of stages, <code>pairId</code> (stable for one pair of runs; key on it), <code>baseStageIds</code> and <code>candStageIds</code>, <code>quality</code> (<code>exact</code>, <code>structural</code> or <code>aligned</code>), <code>score</code> (0 to 1) and <code>deltas</code>. <code>deltas</code> has one <code>{ baseline, candidate, delta }</code> entry for each of <code>executorRunTime</code>, <code>executorCpuTime</code> (ms), <code>memoryBytesSpilled</code>, <code>diskBytesSpilled</code>, <code>inputBytes</code>, <code>outputBytes</code>, <code>shuffleReadBytes</code> and <code>shuffleWriteBytes</code>. The figures count every task attempt of the stage, failed ones included, the same sums the whole-run <code>metrics</code> use; a figure the log did not record is <code>null</code>. A stage&#39;s own record in the run report keeps the latest attempt&#39;s figures.</li><li><code>unmatched</code>: <code>{ baseStageIds, candStageIds }</code>, the stages that paired with nothing.</li><li><code>replanned</code>: groups of stages whose plan changed between the runs. Always an empty array.</li><li><code>bookkeepingStageIds</code>: <code>{ baseStageIds, candStageIds }</code>, stages that only read the Delta log or its checkpoints. They are in no pair, not in <code>unmatched</code> and not in the coverage.</li><li><code>runtimeCoverage</code>: the share of both runs&#39; executor run time (bookkeeping stages excluded) in paired stages, or <code>null</code> when neither run recorded any.</li><li><code>matchedCoverage</code>: the share of stages paired by exact match, by count.</li></ul><p>Every pair has the quality <code>exact</code>; <code>structural</code> and <code>aligned</code> are reserved values. See <a href="./../run-comparison/how-stages-are-matched.html">How stages are matched</a>.</p><p><code>confidence</code> is <code>ok</code>, <code>low</code> or <code>insufficient</code>, and <code>reason</code> says why when it is not <code>ok</code>. It is <code>low</code> when the two runs&#39; names differ, or when under 90% of executor run time is in paired stages (<code>runtimeCoverage</code>), for example &quot;Only 62% of executor run time is in matched stages&quot;. It is <code>insufficient</code> when neither run recorded any executor run time.</p>`,15)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each <code>findings.introduced</code>/<code>resolved</code> row is <code>{ rule, type, impactBand, baseCount, candCount, delta, stages }</code>, with <code>stages</code> naming the affected stages. A <code>metrics</code> row&#39;s <code>direction</code> is <code>improvement</code>, <code>regression</code>, <code>unchanged</code>, <code>neutral</code> (a volume metric, where more isn&#39;t worse) or <code>unavailable</code>, and an unavailable row carries an <code>unavailableReason</code>.</p><p>The comparison block (<code>comparisonSchemaVersion</code> <code>1</code>) holds:</p><ul><li><code>stagePairs</code> (only with <code>include</code>): one entry per pair of stages, <code>pairId</code> (stable for one pair of runs; key on it), <code>baseStageIds</code> and <code>candStageIds</code>, <code>quality</code> (<code>exact</code>, <code>structural</code> or <code>aligned</code>), <code>score</code> (0 to 1) and <code>deltas</code>. <code>deltas</code> has one <code>{ baseline, candidate, delta }</code> entry for each of <code>executorRunTime</code>, <code>executorCpuTime</code> (ms), <code>memoryBytesSpilled</code>, <code>diskBytesSpilled</code>, <code>inputBytes</code>, <code>outputBytes</code>, <code>shuffleReadBytes</code> and <code>shuffleWriteBytes</code>. The figures count every task attempt of the stage, failed ones included, the same sums the whole-run <code>metrics</code> use; a figure the log did not record is <code>null</code>. A stage&#39;s own record in the run report keeps the latest attempt&#39;s figures.</li><li><code>unmatched</code>: <code>{ baseStageIds, candStageIds }</code>, the stages that paired with nothing and sit in no replanned group.</li><li><code>replanned</code>: one group for each aligned pair of SQL executions whose stage counts differ and that left stages unpaired (a broadcast join that took out an exchange, for example). A group holds <code>baseExecutionId</code>, <code>candExecutionId</code>, <code>baseStageIds</code> and <code>candStageIds</code> (the leftover stages per side) and <code>deltas</code>, the total of each of the eight delta metrics over the leftover stages of each side, as <code>{ baseline, candidate, delta }</code> entries like a pair&#39;s. Leftover stages under an execution pair with equal stage counts are <code>unmatched</code>.</li><li><code>bookkeepingStageIds</code>: <code>{ baseStageIds, candStageIds }</code>, stages that only read the Delta log or its checkpoints. They are in no pair, not in <code>unmatched</code> and not in the coverage.</li><li><code>runtimeCoverage</code>: the share of both runs&#39; executor run time (bookkeeping stages excluded) in paired and replanned stages, or <code>null</code> when neither run recorded any.</li><li><code>executionAlignment</code>: how the SQL executions of the two runs lined up: <code>baseExecutions</code> and <code>candExecutions</code> (executions with at least one stage outside the Delta bookkeeping set), <code>pairedExecutions</code>, <code>agreement</code> (<code>pairedExecutions / min(baseExecutions, candExecutions)</code>, <code>null</code> when either run has none), <code>accepted</code> (false when <code>agreement</code> is under 0.5: the runs share too little SQL work to be one job, and no stage pairs) and <code>bounded</code> (true when the alignment ran in its band form, which happens above 1,000,000 execution pairs).</li><li><code>matchedCoverage</code>: the share of stages paired by the exact key alone, by count.</li></ul><p>A pair&#39;s <code>quality</code> is <code>exact</code> (same normalized name and plan text), <code>structural</code> (same plan shape and attribute names, different text) or <code>aligned</code> (different structure with similar text, or no plan to compare and paired by position among stages of one name). <code>score</code> is 1 for <code>exact</code>, the text similarity for the other two, and 0.5 for a pair made by position. See <a href="./../run-comparison/how-stages-are-matched.html">How stages are matched</a>.</p><p><code>confidence</code> is <code>ok</code>, <code>low</code> or <code>insufficient</code>, and <code>reason</code> says why when it is not <code>ok</code>. It is <code>low</code> when the two runs&#39; names differ, or when under 90% of executor run time is in paired stages (<code>runtimeCoverage</code>), for example &quot;Only 62% of executor run time is in matched stages&quot;. It is <code>insufficient</code> when neither run recorded any executor run time.</p>`,15)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};

package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.CPowlb2q.lean.js → user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js}

@@ -1 +1 @@
-import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"compare_runs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/compare-runs.md","filePath":"user-guide/mcp-tools/compare-runs.md"}'),n={name:"user-guide/mcp-tools/compare-runs.md"};function h(l,s,o,d,p,k){return a(),e("div",null,[...s[0]||(s[0]=[t("",15)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};
+import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"compare_runs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/compare-runs.md","filePath":"user-guide/mcp-tools/compare-runs.md"}'),n={name:"user-guide/mcp-tools/compare-runs.md"};function h(o,s,l,d,p,k){return a(),e("div",null,[...s[0]||(s[0]=[t("",15)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};

package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.Dq5Azk8G.js → user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js}

@@ -1,4 +1,4 @@
-import{_ as e,o as i,c as a,a5 as t}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"evaluate_budgets","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/evaluate-budgets.md","filePath":"user-guide/mcp-tools/evaluate-budgets.md"}'),n={name:"user-guide/mcp-tools/evaluate-budgets.md"};function o(d,s,l,h,c,r){return i(),a("div",null,[...s[0]||(s[0]=[t(`<h1 id="evaluate-budgets" tabindex="-1"><code>evaluate_budgets</code> <a class="header-anchor" href="#evaluate-budgets" aria-label="Permalink to &quot;\`evaluate_budgets\`&quot;">​</a></h1><p>Reference for the <code>evaluate_budgets</code> tool of the SparkForensics MCP server.</p><p>Evaluate a run against pass/fail thresholds: the MCP equivalent of the CLI&#39;s <code>--max-runtime</code>/<code>--max-spill</code>/etc. budget flags, thin-wrapped over the same <code>evaluateBudgets()</code> the <code>sparkforensics-analyze</code> CLI uses.</p><p>Parameters (all optional):</p><ul><li><code>source</code> / <code>runId</code>: identify the run to evaluate (same shape as <code>diagnose_run</code>). When <code>runIdB</code>/<code>sourceB</code> is also given, this run is the regression baseline instead.</li><li><code>maxRuntimeMs</code>, <code>maxSpillGb</code>, <code>maxSkewRatio</code>, <code>maxFailedTaskRatePct</code>, <code>minEfficiencyPct</code>: absolute budgets, each evaluated only if provided. With two runs they apply to the candidate (<code>runIdB</code>/<code>sourceB</code>), the same as the CLI&#39;s <code>--baseline</code> mode. <code>minEfficiencyPct</code> measures busy core time, the share of executor core time that ran tasks (100 minus the dashboard&#39;s Unused core time), not the dashboard&#39;s Efficiency tile. <code>maxFailedTaskRatePct</code> reads the task failure rate off the <code>jobFailureRate</code> finding, so when that finding didn&#39;t fire (under 10% of jobs failed, by default) it passes whatever the task failure rate.</li><li><code>runIdB</code> / <code>sourceB</code>: a candidate run to compare against the first, so regression budgets can be evaluated. Same <code>runId</code>/<code>source</code> shape, resolved the same way. Omit both to skip regression budgets.</li><li><code>maxRegressionPct</code>, <code>regressionMetric</code>: fail if <code>regressionMetric</code> (default <code>wallClock</code>; see <a href="./../getting-started/regression-metric-keys.html#regression-metric-keys">Regression metric keys</a> for the full list) regressed by more than <code>maxRegressionPct</code>% between the first run (baseline) and the second run (candidate). Requires <code>runIdB</code>/<code>sourceB</code>. <code>regressionMetric</code> without <code>maxRegressionPct</code> fails the call. A volume key (<code>inputBytes</code>, <code>outputBytes</code>, <code>taskCount</code>, <code>executorsAdded</code>), which has no regression direction, or an unknown key reports <code>inconclusive</code>.</li><li><code>regressionBudgets</code>: further regression budgets, an array of <code>{ &quot;metric&quot;: &quot;&lt;key&gt;&quot;, &quot;maxPct&quot;: &lt;number&gt; }</code>, the MCP form of the CLI&#39;s repeated <code>--regression-budget &lt;metric&gt;:&lt;pct&gt;</code>. Each is checked like <code>maxRegressionPct</code> and reports its own <code>max-regression</code> result. A metric can be budgeted once across this list and the <code>maxRegressionPct</code>/<code>regressionMetric</code> pair (which counts as one budget); a repeat fails the call. Requires <code>runIdB</code>/<code>sourceB</code>, as <code>maxRegressionPct</code> does.</li><li><code>failOnIntroduced</code>: fail if the second run introduces any finding in the given impact band (<code>&quot;all&quot;</code> or one of the impact band names). Requires <code>runIdB</code>/<code>sourceB</code>. A band name it doesn&#39;t recognize reports <code>inconclusive</code>.</li></ul><p>A budget whose required evidence is missing (e.g. no <code>runIdB</code>/<code>sourceB</code> for a regression budget, or a run with no trustworthy task-level evidence) reports <code>inconclusive</code>, not a false pass. The tool has no <code>redact</code> input: the budgets read figures and impact bands, never the stage text redaction rewrites, so a redacted comparison gives the same results. Independent of which budgets you pass, an evaluated run with no ApplicationEnd event adds a <code>run-complete</code> result with status <code>inconclusive</code>, the same check that makes the CLI exit <code>3</code>. With two runs, the check applies to the candidate.</p><p>Each result&#39;s <code>name</code> is one of <code>max-runtime</code>, <code>max-spill</code>, <code>max-skew</code>, <code>max-failed-task-rate</code>, <code>min-efficiency</code>, <code>max-regression</code>, <code>fail-on-introduced</code> and <code>run-complete</code>. A <code>max-regression</code> result also carries <code>metric</code>, the key it checked. The returned <code>runId</code> is the first run&#39;s (the baseline, when two runs are given).</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+import{_ as s,o as i,c as a,a5 as t}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"evaluate_budgets","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/evaluate-budgets.md","filePath":"user-guide/mcp-tools/evaluate-budgets.md"}'),n={name:"user-guide/mcp-tools/evaluate-budgets.md"};function o(d,e,c,l,r,h){return i(),a("div",null,[...e[0]||(e[0]=[t(`<h1 id="evaluate-budgets" tabindex="-1"><code>evaluate_budgets</code> <a class="header-anchor" href="#evaluate-budgets" aria-label="Permalink to &quot;\`evaluate_budgets\`&quot;">​</a></h1><p>Reference for the <code>evaluate_budgets</code> tool of the SparkForensics MCP server.</p><p>Evaluate a run against pass/fail thresholds: the MCP equivalent of the CLI&#39;s <code>--max-runtime</code>/<code>--max-spill</code>/etc. budget flags, thin-wrapped over the same <code>evaluateBudgets()</code> the <code>sparkforensics-analyze</code> CLI uses.</p><p>Parameters (all optional):</p><ul><li><code>source</code> / <code>runId</code>: identify the run to evaluate (same shape as <code>diagnose_run</code>). When <code>runIdB</code>/<code>sourceB</code> is also given, this run is the regression baseline instead.</li><li><code>maxRuntimeMs</code>, <code>maxSpillGb</code>, <code>maxSkewRatio</code>, <code>maxFailedTaskRatePct</code>, <code>minEfficiencyPct</code>: absolute budgets, each evaluated only if provided. With two runs they apply to the candidate (<code>runIdB</code>/<code>sourceB</code>), the same as the CLI&#39;s <code>--baseline</code> mode. <code>minEfficiencyPct</code> measures busy core time, the share of executor core time that ran tasks (100 minus the dashboard&#39;s Unused core time), not the dashboard&#39;s Efficiency tile. <code>maxFailedTaskRatePct</code> reads the task failure rate off the <code>jobFailureRate</code> finding, so when that finding didn&#39;t fire (under 10% of jobs failed, by default) it passes whatever the task failure rate.</li><li><code>runIdB</code> / <code>sourceB</code>: a candidate run to compare against the first, so regression budgets can be evaluated. Same <code>runId</code>/<code>source</code> shape, resolved the same way. Omit both to skip regression budgets.</li><li><code>maxRegressionPct</code>, <code>regressionMetric</code>: fail if <code>regressionMetric</code> (default <code>wallClock</code>; see <a href="./../getting-started/regression-metric-keys.html#regression-metric-keys">Regression metric keys</a> for the full list) regressed by more than <code>maxRegressionPct</code>% between the first run (baseline) and the second run (candidate). Requires <code>runIdB</code>/<code>sourceB</code>. <code>regressionMetric</code> without <code>maxRegressionPct</code> fails the call. A volume key (<code>inputBytes</code>, <code>outputBytes</code>, <code>taskCount</code>, <code>executorsAdded</code>), which has no regression direction, or an unknown key reports <code>inconclusive</code>.</li><li><code>regressionBudgets</code>: further regression budgets, an array of <code>{ &quot;metric&quot;: &quot;&lt;key&gt;&quot;, &quot;maxPct&quot;: &lt;number&gt; }</code>, the MCP form of the CLI&#39;s repeated <code>--regression-budget &lt;metric&gt;:&lt;pct&gt;</code>. Each is checked like <code>maxRegressionPct</code> and reports its own <code>max-regression</code> result. A metric can be budgeted once across this list and the <code>maxRegressionPct</code>/<code>regressionMetric</code> pair (which counts as one budget); a repeat fails the call. Requires <code>runIdB</code>/<code>sourceB</code>, as <code>maxRegressionPct</code> does.</li><li><code>stageRegressionBudgets</code>: per paired stage budgets, an array of <code>{ &quot;metric&quot;: &quot;&lt;key&gt;&quot;, &quot;maxPct&quot;: &lt;number&gt; }</code>, the MCP form of the CLI&#39;s repeated <code>--stage-regression-budget &lt;metric&gt;:&lt;pct&gt;</code>. <code>metric</code> is one of <code>executorRunTime</code>, <code>executorCpuTime</code>, <code>memoryBytesSpilled</code>, <code>diskBytesSpilled</code>, <code>shuffleReadBytes</code> and <code>shuffleWriteBytes</code>. The budget fails when any paired stage&#39;s metric grew by more than <code>maxPct</code>% against its baseline stage; growth from a zero baseline always exceeds it. The detail names the worst pairs by <code>pairId</code> (the same id <code>compare_runs</code> returns in <code>stagePairs</code>). Each metric gets its own <code>max-stage-regression</code> result, and a metric can be budgeted once. Requires <code>runIdB</code>/<code>sourceB</code>. It reports <code>inconclusive</code> when no stage pair is eligible or the metric is missing on every eligible pair. <code>inputBytes</code> and <code>outputBytes</code> are workload volume with no regression direction, so the call fails if either is named.</li><li><code>stageQualities</code>: the pair qualities <code>stageRegressionBudgets</code> read, from <code>exact</code>, <code>structural</code> and <code>aligned</code> (default <code>exact</code> and <code>structural</code>). An <code>aligned</code> pair was matched on similarity or position and may compare different work, so it is left out unless named. Requires <code>stageRegressionBudgets</code>. Stages a re-plan left over are not paired and are not read.</li><li><code>failOnIntroduced</code>: fail if the second run introduces any finding in the given impact band (<code>&quot;all&quot;</code> or one of the impact band names). Requires <code>runIdB</code>/<code>sourceB</code>. A band name it doesn&#39;t recognize reports <code>inconclusive</code>.</li></ul><p>A budget whose required evidence is missing (e.g. no <code>runIdB</code>/<code>sourceB</code> for a regression budget, or a run with no trustworthy task-level evidence) reports <code>inconclusive</code>, not a false pass. The tool has no <code>redact</code> input: the budgets read figures and impact bands, never the stage text redaction rewrites, so a redacted comparison gives the same results. Independent of which budgets you pass, an evaluated run with no ApplicationEnd event adds a <code>run-complete</code> result with status <code>inconclusive</code>, the same check that makes the CLI exit <code>3</code>. With two runs, the check applies to the candidate.</p><p>Each result&#39;s <code>name</code> is one of <code>max-runtime</code>, <code>max-spill</code>, <code>max-skew</code>, <code>max-failed-task-rate</code>, <code>min-efficiency</code>, <code>max-regression</code>, <code>max-stage-regression</code>, <code>fail-on-introduced</code> and <code>run-complete</code>. A <code>max-regression</code> or <code>max-stage-regression</code> result also carries <code>metric</code>, the key it checked. The returned <code>runId</code> is the first run&#39;s (the baseline, when two runs are given).</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;evaluate_budgets&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;baseline.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
@@ -23,4 +23,4 @@ import{_ as e,o as i,c as a,a5 as t}from"./chunks/framework.DSg0KOwT.js";const k
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;violated&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;inconclusive&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div>`,11)])])}const u=e(n,[["render",o]]);export{k as __pageData,u as default};
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div>`,11)])])}const u=s(n,[["render",o]]);export{k as __pageData,u as default};

package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.Dq5Azk8G.lean.js → user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js}

@@ -1 +1 @@
-import{_ as e,o as i,c as a,a5 as t}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"evaluate_budgets","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/evaluate-budgets.md","filePath":"user-guide/mcp-tools/evaluate-budgets.md"}'),n={name:"user-guide/mcp-tools/evaluate-budgets.md"};function o(d,s,l,h,c,r){return i(),a("div",null,[...s[0]||(s[0]=[t("",11)])])}const u=e(n,[["render",o]]);export{k as __pageData,u as default};
+import{_ as s,o as i,c as a,a5 as t}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"evaluate_budgets","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/evaluate-budgets.md","filePath":"user-guide/mcp-tools/evaluate-budgets.md"}'),n={name:"user-guide/mcp-tools/evaluate-budgets.md"};function o(d,e,c,l,r,h){return i(),a("div",null,[...e[0]||(e[0]=[t("",11)])])}const u=s(n,[["render",o]]);export{k as __pageData,u as default};

package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.js}

@@ -1,4 +1,4 @@
-import{_ as i,o as a,c as t,a5 as n}from"./chunks/framework.DSg0KOwT.js";const E=JSON.parse('{"title":"get_run_summary","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/get-run-summary.md","filePath":"user-guide/mcp-tools/get-run-summary.md"}'),e={name:"user-guide/mcp-tools/get-run-summary.md"};function h(k,s,l,p,o,r){return a(),t("div",null,[...s[0]||(s[0]=[n(`<h1 id="get-run-summary" tabindex="-1"><code>get_run_summary</code> <a class="header-anchor" href="#get-run-summary" aria-label="Permalink to &quot;\`get_run_summary\`&quot;">​</a></h1><p>Reference for the <code>get_run_summary</code> tool of the SparkForensics MCP server.</p><p>App/stage/job/sql counts, duration, and how the run ended: no findings. <code>failedJobs</code>/<code>totalJobs</code> count only jobs with an end record, and <code>failureReason</code> is the first line of Spark&#39;s own recorded reason when a job failed, the same line the dashboard verdict quotes (<code>failureReasonStageId</code> is the stage it came from, or <code>null</code> when there is no reason or it came from a job&#39;s exception). <code>runShape</code> carries the dashboard&#39;s run-shape figures: <code>wallClockMs</code>, <code>efficiencyPct</code> (the share of the run with a stage running, the Scorecard&#39;s Efficiency), <code>unusedCoreTimePct</code> (the share of executor core time that ran no task, against peak concurrent cores; <code>minEfficiencyPct</code> checks 100 minus this), <code>etlPhasesMs</code> (<code>extract</code>/<code>transform</code>/<code>load</code> summed stage time, so a phase can exceed the run) and <code>peakBusyCores</code> (busy cores at the peak of Core Usage by Locality, averaged over one chart bucket, so it can be fractional). Each is <code>null</code> where the dashboard shows &quot;Not measured&quot; or &quot;Unavailable&quot;.</p><p>Parameters: <code>source</code>, <code>runId</code> and <code>redact</code>, as in <code>diagnose_run</code>.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;get_run_summary&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;app-20260101.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } } }</span></span></code></pre></div><p>Example response:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+import{_ as i,o as a,c as t,a5 as n}from"./chunks/framework.DSg0KOwT.js";const E=JSON.parse('{"title":"get_run_summary","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/get-run-summary.md","filePath":"user-guide/mcp-tools/get-run-summary.md"}'),e={name:"user-guide/mcp-tools/get-run-summary.md"};function h(k,s,l,p,o,r){return a(),t("div",null,[...s[0]||(s[0]=[n(`<h1 id="get-run-summary" tabindex="-1"><code>get_run_summary</code> <a class="header-anchor" href="#get-run-summary" aria-label="Permalink to &quot;\`get_run_summary\`&quot;">​</a></h1><p>Reference for the <code>get_run_summary</code> tool of the SparkForensics MCP server.</p><p>App/stage/job/sql counts, duration, and how the run ended: no findings. <code>failedJobs</code>/<code>totalJobs</code> count only jobs with an end record, and <code>failureReason</code> is the first line of Spark&#39;s own recorded reason when a job failed, the same line the dashboard verdict quotes (<code>failureReasonStageId</code> is the stage it came from, or <code>null</code> when there is no reason or it came from a job&#39;s exception). <code>runShape</code> carries the dashboard&#39;s run-shape figures: <code>wallClockMs</code>, <code>efficiencyPct</code> (the share of the run with a stage running, the Scorecard&#39;s Efficiency), <code>unusedCoreTimePct</code> (the share of executor core time that ran no task, against the run&#39;s allocation, cores times the time each executor was alive; <code>minEfficiencyPct</code> checks 100 minus this), <code>etlPhasesMs</code> (<code>extract</code>/<code>transform</code>/<code>load</code> summed stage time, so a phase can exceed the run) and <code>peakBusyCores</code> (busy cores at the peak of Core Usage by Locality, averaged over one chart bucket, so it can be fractional). Each is <code>null</code> where the dashboard shows &quot;Not measured&quot; or &quot;Unavailable&quot;.</p><p>Parameters: <code>source</code>, <code>runId</code> and <code>redact</code>, as in <code>diagnose_run</code>.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;get_run_summary&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;app-20260101.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } } }</span></span></code></pre></div><p>Example response:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;runId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;3f9c2b7e-...&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;app&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;id&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;application_0000000000000_0001&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;t&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;sparkVersion&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;stageCount&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>

package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js

@@ -0,0 +1 @@
+import{_ as t,o as a,c as o,a5 as n}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Run comparison mode","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/run-comparison.md","filePath":"user-guide/run-comparison.md"}'),r={name:"user-guide/run-comparison.md"};function s(i,e,d,h,c,l){return a(),o("div",null,[...e[0]||(e[0]=[n('<h1 id="run-comparison-mode" tabindex="-1">Run comparison mode <a class="header-anchor" href="#run-comparison-mode" aria-label="Permalink to &quot;Run comparison mode&quot;">​</a></h1><p>Put two runs side by side to see whether a tuning change helped.</p><h2 id="starting-a-comparison" tabindex="-1">Starting a comparison <a class="header-anchor" href="#starting-a-comparison" aria-label="Permalink to &quot;Starting a comparison&quot;">​</a></h2><p>From the landing page, click <strong>Compare two runs</strong>. Two slots appear: <strong>Baseline</strong> and <strong>Candidate</strong>.</p><ol><li>Load a run into each slot the same way you&#39;d load a single run (drag and drop, <strong>Choose file</strong>, <strong>Choose rolling-log folder</strong>, or a History Server fetch).</li><li>Click <strong>Compare</strong>.</li></ol><p>The runs parse one after the other through the same background worker. When both finish, the comparison view opens.</p><p>Already looking at a run, for example the one before a change? Click <strong>Compare with another run</strong> in the top bar (or the <strong>More options</strong> menu on a phone). The compare view opens with that run as the baseline, so you only load the candidate, and the open run is not parsed again. <strong>Back to the run</strong> returns to its dashboard.</p><h2 id="reading-the-comparison" tabindex="-1">Reading the comparison <a class="header-anchor" href="#reading-the-comparison" aria-label="Permalink to &quot;Reading the comparison&quot;">​</a></h2><p>The page opens with a verdict: whether the candidate finished faster or slower than the baseline, and by how much (a change under 2% reads as &quot;about as long&quot;). When either run had failed jobs, the verdict leads with that instead, for example &quot;The candidate had 2 of 5 jobs fail (baseline: none)&quot; or &quot;The candidate&#39;s only job failed&quot;, and the run-time line follows. When either log has no end-of-run record (the run was killed or the log cut off), the verdict says how much run time each log covers, in a neutral tone, instead of calling the shorter run faster. It then lists which cost metrics got worse or better in the candidate (again ignoring changes under 2%), and which finding categories became more or less frequent. Volume and count metrics (input, output, tasks, executors) are left out, since more or less of them is not better or worse on its own. <strong>See where to start in the candidate</strong> opens the candidate&#39;s dashboard, whose own verdict names the first thing to fix.</p><p>The <strong>Metrics</strong> table covers the whole run: wall-clock duration, memory spill, task skew, failed-task rate, disk spill, GC time, input/output bytes, executor run-time, and task/executor counts, baseline against candidate with the change. <strong>Findings by category</strong> lists which finding types became more or less frequent in the candidate, per impact level, with the count in each run and the affected stages.</p><p>The <strong>Stages compared</strong> table lists every paired stage with its change in run time, CPU time, spill (memory and disk together), input, output and shuffle (read and write together), largest run-time change first. A faster candidate reads green and a slower one red; input and output are workload volume, so they stay neutral. Under each stage name sit the pair&#39;s quality (<code>exact</code>, <code>structural</code> or <code>aligned</code>) and its score from 0 to 1. An <code>aligned</code> pair was matched on similarity or position, so treat its change with care.</p><p>Select a stage&#39;s name to see both runs&#39; figures for it side by side without leaving the comparison: the stage&#39;s name in each run, then duration, run time, GC time, spill, input, output and task counts with the change. <strong>Open stage N in the baseline dashboard</strong> and <strong>Open stage N in the candidate dashboard</strong> open that stage&#39;s detail in the run&#39;s own dashboard, and <strong>← Back to comparison</strong> returns here. Below the table, <strong>Re-planned work</strong> lists queries that ran a different number of stages in the two runs, with the leftover stages and their total run time per side, and <strong>Unmatched stages</strong> lists stages that paired with nothing. Both open in the same side-by-side view. The table shows 25 pairs and reveals more on request.</p><p>The table columns read <strong>Baseline</strong> and <strong>Candidate</strong>; hover a column heading to see the full name of the run.</p><p>A banner above the verdict appears when the match is weak. It reads <code>low</code> when under 90% of executor run time sits in paired stages (or the application names differ), and <code>insufficient</code> when neither run recorded any executor run time, so there is nothing to pair. The banner can be dismissed; every change on the page is still shown.</p><p>How stages are paired, and what the matching confidence means, is in <a href="./run-comparison/how-stages-are-matched.html">How stages are matched</a>.</p><p>For everything else, matching is manual: the <strong>Pinned per-stage deltas</strong> widget lets you pick one stage from the baseline and one from the candidate yourself and pin the pair, then shows their duration, executor run-time, GC time, memory/disk spill, input/output bytes, task count, and failed tasks side by side. Pin as many pairs as you want to check.</p><p>For either run&#39;s full dashboard, click <strong>View baseline dashboard</strong> or <strong>View candidate dashboard</strong>. <strong>← Back to comparison</strong> takes you back.</p><h2 id="comparing-without-the-dashboard" tabindex="-1">Comparing without the dashboard <a class="header-anchor" href="#comparing-without-the-dashboard" aria-label="Permalink to &quot;Comparing without the dashboard&quot;">​</a></h2><p>The same comparison runs headlessly, and opens with the same verdict: the positional run is the candidate, the <code>--baseline</code> run the baseline. The CLI&#39;s <code>--baseline &lt;local file or rolling-log dir&gt;</code> flag adds the comparison. In JSON the run&#39;s report moves under <code>candidate</code> and the comparison sits beside it under <code>comparison</code> (<code>comparison.verdict</code> is the headline); in Markdown the comparison section follows the report. <code>--baseline</code> can also gate a build on the comparison (<code>--max-regression-pct</code>, <code>--stage-regression-budget</code>, <code>--fail-on-introduced</code>; see <a href="./getting-started/ci-and-automation.html#ci-and-automation">Getting started</a>). The MCP server&#39;s <code>compare_runs</code> and <code>evaluate_budgets</code> tools do the same for an AI assistant; see <a href="./mcp-tools.html">MCP tools reference</a>.</p>',19)])])}const p=t(r,[["render",s]]);export{g as __pageData,p as default};

package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js

@@ -0,0 +1 @@
+import{_ as t,o as a,c as o,a5 as n}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Run comparison mode","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/run-comparison.md","filePath":"user-guide/run-comparison.md"}'),r={name:"user-guide/run-comparison.md"};function s(i,e,d,h,c,l){return a(),o("div",null,[...e[0]||(e[0]=[n("",19)])])}const p=t(r,[["render",s]]);export{g as __pageData,p as default};

package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.DSXGu2tA.js

@@ -0,0 +1 @@
+import{_ as t,o as a,c as n,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"How stages are matched","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/run-comparison/how-stages-are-matched.md","filePath":"user-guide/run-comparison/how-stages-are-matched.md"}'),s={name:"user-guide/run-comparison/how-stages-are-matched.md"};function i(r,e,d,c,h,l){return a(),n("div",null,[...e[0]||(e[0]=[o('<h1 id="how-stages-are-matched" tabindex="-1">How stages are matched <a class="header-anchor" href="#how-stages-are-matched" aria-label="Permalink to &quot;How stages are matched&quot;">​</a></h1><p>How SparkForensics pairs stages between a baseline and a candidate, and how sure it is about the pairing.</p><p>Stage-level detail depends on pairing a stage in the baseline with its counterpart in the candidate. A stage ID is no help: AQE&#39;s runtime replanning and a different submission order change it from run to run. So a stage is identified by what it ran, and pairing happens in two levels: first the SQL executions of the two runs line up, then the stages inside each pair of executions.</p><h2 id="text-that-differs-between-runs" tabindex="-1">Text that differs between runs <a class="header-anchor" href="#text-that-differs-between-runs" aria-label="Permalink to &quot;Text that differs between runs&quot;">​</a></h2><p>Before two stages are compared, the text that differs between two runs of the same job is rewritten to a fixed form:</p><ul><li>random per-run staging directories in paths (a 6-8 character segment that mixes cases, or letters with digits: a real directory named <code>output01</code> is rewritten too)</li><li>dates and timestamps, such as the run date in a predicate</li><li><code>IN</code> and <code>INSET</code> lists, which grow with the processing window</li><li>file counts in a file index, such as <code>DeltaLogFileIndex(4 paths)</code></li></ul><p>Numbers, paths and string literals in general are not rewritten: two stages that differ in a filter constant or an input path have different exact keys (see below). Text that is specific to your runs, such as an output directory that includes the run&#39;s variant name, can be rewritten with the CLI&#39;s <code>--normalize-path</code> flag or the MCP <code>compare_runs</code> tool&#39;s <code>normalizePath</code> parameter. Despite the name it is a generic regular expression, not only for paths, and it affects only which stages pair, never the findings.</p><h2 id="level-one-sql-executions" tabindex="-1">Level one: SQL executions <a class="header-anchor" href="#level-one-sql-executions" aria-label="Permalink to &quot;Level one: SQL executions&quot;">​</a></h2><p>Each run&#39;s SQL executions are laid out in submission order and aligned with an order-preserving sequence alignment (a longest-common-subsequence dynamic program). An execution pair scores on three things: the call site (the user-code frames in the stages&#39; stack traces), the description, and the plan structure (operator names and the attribute names the plan mentions). A part one side lacks drops out of the score, which matters for PySpark, where every call site is generic. Only pairs scoring at least 0.55 can match, and of equal alignments the one that keeps the lower execution IDs wins, so the result is deterministic and swapping baseline and candidate mirrors it (except when two crossing alignments tie exactly, where the answer depends on which run is the baseline).</p><p>A loop of identical queries that ran 14 times in one run and 15 in the other pairs 14 iterations and leaves the extra iteration unmatched.</p><p>Stages outside any SQL execution (RDD jobs, for example) form one pseudo-execution per run, paired with each other and matched on stage name and call-site text.</p><p>The alignment needs at least half of the smaller run&#39;s executions to pair (<code>paired / min(baseline executions, candidate executions)</code> is at least 0.5). A candidate that removes work, such as caching that drops repeated queries, still pairs everything it kept. Below that the two runs are not treated as one job and no stage pairs: stages as generic as a <code>count</code> or a <code>collect</code> repeat across unrelated jobs, and only the executions around them tell the jobs apart. The comparison reports the counts and the ratio as <code>executionAlignment</code>.</p><h3 id="cost" tabindex="-1">Cost <a class="header-anchor" href="#cost" aria-label="Permalink to &quot;Cost&quot;">​</a></h3><p>The full alignment compares every baseline execution with every candidate execution. Up to 1,000,000 such pairs (1,000 by 1,000 executions) it runs in full. Beyond that it is restricted to a band around the diagonal, about <code>2 * (100 + |n - m| / 2) + 1</code> candidates per baseline execution, so the work grows with the number of executions, not its square. A comparison that used the band reports <code>executionAlignment.bounded: true</code>. An insertion or deletion run inside the band is found; one that shifts the two runs further apart than the band is not, and the executions it separates stay unmatched.</p><h2 id="level-two-stages-inside-an-execution-pair" tabindex="-1">Level two: stages inside an execution pair <a class="header-anchor" href="#level-two-stages-inside-an-execution-pair" aria-label="Permalink to &quot;Level two: stages inside an execution pair&quot;">​</a></h2><p>Every stage has two keys. The <strong>exact key</strong> is its normalized name plus the fingerprint of the plan nodes its tasks were attributed to (normalized node text). The <strong>structural key</strong> is the shape of those nodes: the operator tree, node names and the sorted attribute names each node mentions, with literals, paths, file counts and ids left out. A grouping set that a program emits in a different column order each time has one structural key.</p><p>Inside an aligned execution pair, stages pair in this order, and a stage pairs at most once:</p><ol><li><strong><code>exact</code></strong>: the same exact key. Stages that share a key pair off in stage-ID order, as many as both sides have. A self-join subtree that ran 3 times in one run and 2 in the other pairs 2.</li><li><strong><code>structural</code></strong>: the same structural key, different text.</li><li><strong><code>aligned</code></strong>: different structure, but the text (stage name and normalized node text, compared as sets of tokens) is at least 60% similar. The best scoring pairs go first.</li><li><strong><code>aligned</code></strong>, by position: stages with no attributed plan nodes pair by position among the stages of the same name inside their execution. These pairs have the score 0.5, since no plan evidence backs them.</li></ol><p><code>score</code> is 1 for an exact pair, the text similarity for a structural or aligned one, and 0.5 for a pair made by position. A caller that wants only pairs it can trust reads <code>quality</code>.</p><h2 id="re-planned-executions" tabindex="-1">Re-planned executions <a class="header-anchor" href="#re-planned-executions" aria-label="Permalink to &quot;Re-planned executions&quot;">​</a></h2><p>A configuration change can make Spark plan an execution differently: a broadcast join takes out an exchange, so the two sides run different numbers of stages. When an aligned execution pair has different stage counts (Delta bookkeeping stages excluded), the stages left unpaired on either side are reported together in <code>replanned</code>, with both execution IDs, the leftover stage IDs per side and the total of each delta metric per side. Stages that did pair keep their pair. When the stage counts are equal, leftover stages are <code>unmatched</code> instead: the same number of stages with no pairing point at changed work, not at a re-plan. The stages outside any SQL execution have no plan to re-plan, so their leftovers are <code>unmatched</code>.</p><p>Stages that only read the Delta transaction log or its checkpoints are table bookkeeping, not query work. They are listed apart from the pairs and counted in neither the pairs, the unmatched stages, the replanned stages nor the coverage.</p><h2 id="runtime-coverage-and-confidence" tabindex="-1">Runtime coverage and confidence <a class="header-anchor" href="#runtime-coverage-and-confidence" aria-label="Permalink to &quot;Runtime coverage and confidence&quot;">​</a></h2><p>The comparison reports the <strong>runtime coverage</strong>: the share of both runs&#39; executor run time that sits in paired or replanned stages, with every task attempt counted (failed and speculative ones too). It is weighted by run time, not stage count, because a few unmatched heavy stages matter more than many matched small ones. The share of stages matched, by count, is reported too (<code>matchedCoverage</code>) and does not decide the confidence.</p><p>The confidence is:</p><ul><li><code>ok</code> when the runs have the same application name and the runtime coverage is at least 90%.</li><li><code>low</code> when the application names differ, or the runtime coverage is under 90%. A warning banner says which, for example &quot;Only 62% of executor run time is in matched stages, so per-stage rows mostly compare unrelated work&quot;. The metric deltas above it still hold: they do not depend on stage pairing.</li><li><code>insufficient</code> when neither run recorded any executor run time, so there is no work to compare. The banner says so.</li></ul><p>Replanned run time counts toward the coverage because the two sides of a replanned group still ran the same execution, and the group reports comparable totals. A group shows how much run time it holds, so a coverage made mostly of replanned work is visible in <code>replanned</code>.</p><p>The <strong>Per-stage task skew</strong> table lists only paired stages, and the page states the share of executor run time they hold. Confidence does not change the exit code of a CLI budget.</p>',28)])])}const g=t(s,[["render",i]]);export{p as __pageData,g as default};

package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.DSXGu2tA.lean.js

@@ -0,0 +1 @@
+import{_ as t,o as a,c as n,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"How stages are matched","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/run-comparison/how-stages-are-matched.md","filePath":"user-guide/run-comparison/how-stages-are-matched.md"}'),s={name:"user-guide/run-comparison/how-stages-are-matched.md"};function i(r,e,d,c,h,l){return a(),n("div",null,[...e[0]||(e[0]=[o("",28)])])}const g=t(s,[["render",i]]);export{p as __pageData,g as default};

package/public/docs/assets/user-guide_understanding-findings.md.3C7KCEZO.js

@@ -0,0 +1 @@
+import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Understanding findings","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/understanding-findings.md","filePath":"user-guide/understanding-findings.md"}'),r={name:"user-guide/understanding-findings.md"};function i(n,e,d,c,l,h){return t(),o("div",null,[...e[0]||(e[0]=[s('<h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to &quot;Understanding findings&quot;">​</a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>Each tag links to the matching section of the Spark <a href="./../tuning-reference/">tuning reference</a>, opened in an in-app <strong>Reference</strong> panel next to the board. <code>INCMP</code> has no reference section, so its tag opens this page instead. Turn on <a href="./getting-started/reading-the-dashboard.html#advanced-view">Advanced view</a> to also see a finding&#39;s confidence when it is below high, and a page icon beside the tag that opens this page&#39;s entry. Some tags point into another tag&#39;s reference page because the material overlaps: <code>SFAIL</code> shares <code>FAIL</code>&#39;s section; <code>PART</code> is on the <code>SHFL</code> page, <code>SPEC</code> on <code>STRAG</code>, <code>SHAPE</code> on <code>SKEW</code>, <code>SLOW</code> on <code>HOST</code>, and <code>CACHE</code>/<code>LOCAL</code> on <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to &quot;Per-stage&quot;">​</a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to &quot;`SKEW`: Task skew {#skew}&quot;">​</a></h3><p>A small number of tasks take much longer than their peers in the same stage. The fix depends on what the stage reads, which the finding&#39;s <code>evidence.origin</code> records. A stage that reads a shuffle feeding a join (<code>shuffleJoin</code>) gets AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>), unless the run&#39;s effective conf already has it; otherwise salt the key or repartition on a better key. A stage that reads files with uneven sizes (<code>inputScan</code>) gets compaction of small files or a lower <code>spark.sql.files.maxPartitionBytes</code>. Any other stage (<code>other</code>) gets the salting advice and no conf, as a <code>code</code> entry in <code>remediation</code>; so does a join stage whose skew-join handling is already on. Flagged when P95 task time (the longest task, on a stage with fewer than 20 tasks) exceeds 3x the median and the recoverable tail is at least 0.5% of the run. The median is the textbook one: on an even task count, the mean of the two middle values.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to &quot;`SHFL`: Shuffle I/O {#shfl}&quot;">​</a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join. The partition-count advice follows the effective conf (<code>evidence.partitions</code>): <code>raise</code> when the property limits the stage, <code>sufficient</code> when the tasks already read a good size each, <code>aqeCoalesced</code> when AQE merged the partitions (lower <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>), and <code>ownPartitioning</code> when the property is already high enough (the stage&#39;s own <code>repartition(n)</code> or RDD parallelism limits it). Only flagged on stages that take at least 0.5% of the run.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to &quot;`SPILL`: Memory and disk spill {#spill}&quot;">​</a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don&#39;t (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions or executor memory). <code>spark.sql.shuffle.partitions</code> is suggested only for a stage that reads a shuffle; <code>evidence.reads</code> says what the stage reads (<code>shuffle</code>, <code>input</code> or <code>other</code>). Only flagged on stages that take at least 0.5% of the run.</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to &quot;`GC`: Garbage collection pressure {#gc}&quot;">​</a></h3><p>Tasks spend more than 10% of executor run time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory. A stage with GC below 5% gets an informational note that executor memory may be over-provisioned, only on stages that take at least 0.5% of the run. Both need at least 10 s of executor run time on the stage.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to &quot;`FAIL`: Failed tasks {#fail}&quot;">​</a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors. The finding names the dominant error: the exception class, or the executor loss reason (for example &quot;Container killed by YARN for exceeding memory limits&quot;). It lists up to five distinct failures, each with its message and a short stack excerpt. With redaction on, messages become <code>[redacted]</code> and message lines inside excerpts are dropped, since they can carry file paths and data values; class names, stack frames and the executor loss reason stay (hosts in it are pseudonymized).</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to &quot;`SFAIL`: Failed stage {#sfail}&quot;">​</a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it. With redaction on, the failure reason becomes <code>[redacted]</code>, since it can carry file paths and data values.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to &quot;`STRAG`: Straggler tasks {#strag}&quot;">​</a></h3><p>A few tasks run much slower than the rest of their stage. Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue. If uneven data is the cause, the advice is the one a skew finding gives for the same stage (<code>evidence.origin</code>, see <code>SKEW</code>): AQE skew-join handling for a shuffle feeding a join, file sizes for a scan, salting otherwise. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to &quot;`SPEC`: Speculation waste {#spec}&quot;">​</a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flags a confidence that scales with how far the wasted time sits past the threshold. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to &quot;`RETRY`: Retry waste {#retry}&quot;">​</a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to &quot;`TINY`: Tiny tasks {#tiny}&quot;">​</a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks; a stage that reads a shuffle (<code>evidence.reads</code> is <code>shuffle</code>) can also lower <code>spark.sql.shuffle.partitions</code>. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to &quot;`PART`: Partition sizing {#part}&quot;">​</a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to &quot;`SLOW`: Stage slowness {#slow}&quot;">​</a></h3><p>A stage ran for 15 minutes or more and no slow host was flagged on it. It can appear alongside other findings on the same stage. On a stage that reads a shuffle, often a partition-count problem: raise parallelism via <code>spark.sql.shuffle.partitions</code> or <code>spark.default.parallelism</code>, or check for a large per-task data volume driving heavy shuffle and spill. On a stage that reads input files and no shuffle, check input file sizes and lower <code>spark.sql.files.maxPartitionBytes</code>. <code>evidence.reads</code> says which case applied (<code>shuffle</code>, <code>input</code> or <code>other</code>).</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to &quot;`SHAPE`: Stage shape {#shape}&quot;">​</a></h3><p>The stage has an inefficient task count, output shape (output more than 10× input), or task-to-stage balance: one straggler task running for more than half the stage&#39;s wall-clock time and over 3× the median task, so it alone sets when the stage ends. A too-low task count and a straggler are only flagged on stages that take at least 0.5% of the run.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to &quot;`HOST`: Slow host {#host}&quot;">​</a></h3><p>One executor is much slower than its peers, or carries most of the stage&#39;s task time or bytes. It may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable <code>spark.speculation</code> to relaunch a lagging task automatically. Only flagged on stages that take at least 0.5% of the run.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to &quot;App-level&quot;">​</a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to &quot;`COLD`: Executor cold start {#cold}&quot;">​</a></h3><p>The first stage waited more than 30 s for an executor. Keep a warm pool of executors, or, with dynamic allocation, raise <code>spark.dynamicAllocation.minExecutors</code>/<code>initialExecutors</code> so the app doesn&#39;t scale up from zero. With dynamic allocation off (<code>evidence.dynamicAllocation</code> is <code>off</code>) no executor-count property applies and <code>remediation</code> is empty.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to &quot;`UTIL`: Low utilization {#util}&quot;">​</a></h3><p>Allocated executors sit idle for a large share of the application run: under 60% of the core time the run allocated (cores times the time each executor was alive) ran a task. Consider a smaller cluster. With dynamic allocation off, either lower <code>spark.executor.instances</code> or enable dynamic allocation; the two are alternatives. With it on, lower <code>spark.dynamicAllocation.maxExecutors</code>, and <code>spark.dynamicAllocation.minExecutors</code> too when the run sets it above 0.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to &quot;`MEM`: Memory utilization {#mem}&quot;">​</a></h3><p>Executor memory or core capacity may be over- or under-provisioned: more than 50% of allocated core time ran no task, an executor&#39;s heap peaked above 95% of its allocation, or it stayed below 70%. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can&#39;t be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant self-flags a confidence that scales with how far the estimated waste sits past a 1.5x buffer: it estimates waste from allocated memory-time versus task run time (not measured heap usage). Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to &quot;`CACHE`: Caching opportunity {#cache}&quot;">​</a></h3><p>A reusable dataset (re-read via the same SQL relation more than once, or a join/union result recomputed by two or more executions, matched by plan shape) may be worth persisting between stages. Self-flags a confidence that scales with how many executions reuse the same relation: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to &quot;`CSTOR`: Cache storage {#cstor}&quot;">​</a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><p>The cached-partition counts and sizes come from <code>SparkListenerBlockUpdated</code> events, which Spark writes only when <code>spark.eventLog.logBlockUpdates.enabled=true</code>. Since Spark 2.3 the RDD storage figures in stage-submission events are always 0, and they are used only as a fallback. When a Spark 2.3+ run persists RDDs but its log has neither and block-update logging was off, the check reports that the cache could not be checked instead of passing it: turn <code>spark.eventLog.logBlockUpdates.enabled</code> on and rerun to measure eviction and disk spillover.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to &quot;`LOCAL`: Core usage locality {#local}&quot;">​</a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check executor/data colocation. Self-flags a confidence that scales with the non-local ratio and sample size: the thresholds are our own noise floor for this metric.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to &quot;`CHRN`: Autoscaling churn {#chrn}&quot;">​</a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. With dynamic allocation off (<code>evidence.dynamicAllocation</code> is <code>off</code>) none of those applies: the churn is executor loss or preemption, and <code>remediation</code> is empty. Self-flags a confidence that scales with how far the short-lived-executor share sits past the threshold.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to &quot;`JOBS`: Job failure rate {#jobs}&quot;">​</a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to &quot;`INCMP`: Incomplete run {#incmp}&quot;">​</a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to &quot;Configuration scope&quot;">​</a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to &quot;`CFG`: Configuration audit {#cfg}&quot;">​</a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage&#39;s behavior. Four checks run:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won&#39;t survive executor removal.</li><li><code>spark.dynamicAllocation.minExecutors</code>/<code>maxExecutors</code>: with dynamic allocation on, flagged when min exceeds max (reported on <code>minExecutors</code>) or when no max is set.</li><li><code>spark.serializer</code>: flagged when not set to Kryo (the default is the Java serializer); <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below max(384 MiB, 10% of executor memory).</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to &quot;SQL scope&quot;">​</a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to &quot;`PLAN`: Plan advisor {#plan}&quot;">​</a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan. When the repeats have the same shape but different filters, columns or tables, the finding stays informational and claims no time. Only flagged when the repeat&#39;s stages take at least 0.5% of the run.</li><li>Small files: one plan node reads or writes more than 100 files averaging under 3 MB. Compact upstream output, or coalesce before writing.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>. When the effective threshold (logged, else Spark&#39;s 10 MiB) already admits the smaller side (<code>evidence.broadcastThreshold</code> is <code>notLimiting</code>), the threshold is not what stopped the broadcast, so <code>remediation</code> is empty and the advice is a hint or table statistics.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>. When the effective threshold is below the broadcast or auto-broadcast is disabled (<code>evidence.broadcastThreshold</code> is <code>notLimiting</code> or <code>disabled</code>), a hint forced it: remove the hint, and <code>remediation</code> is empty.</li></ul>',60)])])}const f=a(r,[["render",i]]);export{p as __pageData,f as default};

package/public/docs/assets/user-guide_understanding-findings.md.3C7KCEZO.lean.js

@@ -0,0 +1 @@
+import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Understanding findings","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/understanding-findings.md","filePath":"user-guide/understanding-findings.md"}'),r={name:"user-guide/understanding-findings.md"};function i(n,e,d,c,l,h){return t(),o("div",null,[...e[0]||(e[0]=[s("",60)])])}const f=a(r,[["render",i]]);export{p as __pageData,f as default};