sparkforensics-server 0.4.0 → 0.5.1

package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.js

@@ -0,0 +1,4 @@
+import{_ as t,o,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Finding your event log","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval.md","filePath":"user-guide/alternative-log-retrieval.md"}'),i={name:"user-guide/alternative-log-retrieval.md"};function s(n,e,l,p,d,c){return o(),a("div",null,[...e[0]||(e[0]=[r(`<h1 id="finding-your-event-log" tabindex="-1">Finding your event log <a class="header-anchor" href="#finding-your-event-log" aria-label="Permalink to &quot;Finding your event log&quot;">​</a></h1><p>SparkForensics reads the Spark event log: the newline-delimited JSON file of listener events that Spark writes for each application and that the Spark History Server replays to rebuild the Spark UI. Driver stdout, executor <code>stderr</code> and log4j output are different files and won&#39;t load.</p><p>Spark writes one event log per application when <code>spark.eventLog.enabled</code> is <code>true</code>, into the directory set by <code>spark.eventLog.dir</code> (for example <code>/tmp/spark-events</code>, or an HDFS or object-store path). Copy the file for your application from there. Or download it from a Spark History Server at <code>&lt;history-server&gt;/api/v1/applications/&lt;app-id&gt;/logs</code> and drop the <code>.zip</code> it returns as-is.</p><p>Not sure where your cluster writes it? Open the application&#39;s Spark UI, go to the <strong>Environment</strong> tab and look up <code>spark.eventLog.dir</code>. The sections below cover the usual places, platform by platform.</p><h2 id="spark-you-run-yourself" tabindex="-1">Spark you run yourself <a class="header-anchor" href="#spark-you-run-yourself" aria-label="Permalink to &quot;Spark you run yourself&quot;">​</a></h2><p>Event logging is off unless you turn it on, and it has to be on before the application starts. Pass it at submit time or set it in <code>spark-defaults.conf</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">spark-submit</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --conf</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> spark.eventLog.enabled=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --conf</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> spark.eventLog.dir=file:///tmp/spark-events</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">  my-job.py</span></span></code></pre></div><p>The directory must exist before the job starts. After the job finishes, there is one file per application in it, named after the application ID.</p><p>With <code>spark.eventLog.rolling.enabled=true</code>, Spark writes an <code>eventlog_v2_*</code> directory per application instead, holding numbered <code>events_*</code> parts. Load the whole directory: <strong>Other sources &gt; Choose rolling-log folder</strong> in the browser, or the directory path in the CLI and MCP tools.</p><p>On Kubernetes the driver pod&#39;s own filesystem goes away with the pod, so point <code>spark.eventLog.dir</code> at storage that outlives it: a persistent volume or an object-store path.</p><p>See the Spark docs on <a href="https://spark.apache.org/docs/latest/monitoring.html#viewing-after-the-fact" target="_blank" rel="noreferrer">event logging</a> for every setting.</p><h2 id="from-a-spark-history-server" tabindex="-1">From a Spark History Server <a class="header-anchor" href="#from-a-spark-history-server" aria-label="Permalink to &quot;From a Spark History Server&quot;">​</a></h2><p>A History Server reads its logs from <code>spark.history.fs.logDirectory</code>. If you can read that directory, copy the application&#39;s file or <code>eventlog_v2_*</code> directory straight out of it.</p><p>If you can only reach the History Server over HTTP, download the log instead. The application list&#39;s <strong>Event Log</strong> column has a <strong>Download</strong> link, and <code>GET &lt;history-server&gt;/api/v1/applications/&lt;app-id&gt;/logs</code> returns the same <code>.zip</code>. Drop the <code>.zip</code> as-is. For an application that ran more than once, download one attempt with <code>/api/v1/applications/&lt;app-id&gt;/&lt;attempt-id&gt;/logs</code>.</p><p>SparkForensics can also fetch the log for you when your machine reaches the History Server directly: <strong>Other sources &gt; Fetch from Spark History Server</strong> in <a href="./getting-started.html#local-server-mode">local-server mode</a>, <code>--shs-base-url</code> and <code>--app-id</code> in the <a href="./getting-started/ci-and-automation.html#ci-and-automation">CLI</a>, or a <code>{ shsBaseUrl, appId }</code> source in the <a href="./mcp-tools.html">MCP tools</a>.</p><h2 id="loading-the-file" tabindex="-1">Loading the file <a class="header-anchor" href="#loading-the-file" aria-label="Permalink to &quot;Loading the file&quot;">​</a></h2><p>Once the log is on your machine:</p><ul><li>Browser: drop the file onto the landing page, same as any other run (see <a href="./getting-started.html">Getting started</a>).</li><li>CLI: <code>npx -p sparkforensics-cli sparkforensics-analyze ./application_XXXX_XXXX</code> (full flag list in <a href="./getting-started/ci-and-automation.html#ci-and-automation">Getting started</a>).</li><li>MCP: call a tool with an absolute path, for example <code>{ &quot;source&quot;: { &quot;path&quot;: &quot;/home/me/application_XXXX_XXXX&quot; } }</code> (a relative path resolves against the MCP server&#39;s working directory, which the client chooses; tool reference in <a href="./mcp-tools.html">MCP tools reference</a>).</li></ul><p>The file is the app&#39;s native input format either way: a single event-log file, optionally <code>.gz</code>/<code>.zstd</code>/<code>.lz4</code>/<code>.snappy</code>-compressed, a History Server <code>.zip</code>, or a rolling <code>eventlog_v2_*</code> directory. No conversion step needed: the codec is detected from the file&#39;s contents, not its extension. Spark&#39;s <code>lzf</code> codec is not supported.</p><p>Event logs can hold SQL text, table names, storage paths and host names. Check a log before you share it outside the environment that produced it.</p>`,20)])])}const u=t(i,[["render",s]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.lean.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Finding your event log","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval.md","filePath":"user-guide/alternative-log-retrieval.md"}'),i={name:"user-guide/alternative-log-retrieval.md"};function s(n,e,l,p,d,c){return o(),a("div",null,[...e[0]||(e[0]=[r("",20)])])}const u=t(i,[["render",s]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.js

@@ -0,0 +1,4 @@
+import{_ as a,o as t,c as s,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Managed platforms","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval/managed-platforms.md","filePath":"user-guide/alternative-log-retrieval/managed-platforms.md"}'),o={name:"user-guide/alternative-log-retrieval/managed-platforms.md"};function n(i,e,l,p,d,h){return t(),s("div",null,[...e[0]||(e[0]=[r(`<h1 id="managed-platforms" tabindex="-1">Managed platforms <a class="header-anchor" href="#managed-platforms" aria-label="Permalink to &quot;Managed platforms&quot;">​</a></h1><p>Where to find the Spark event log on each managed platform.</p><h2 id="amazon-emr" tabindex="-1">Amazon EMR <a class="header-anchor" href="#amazon-emr" aria-label="Permalink to &quot;Amazon EMR&quot;">​</a></h2><p>EMR turns event logging on by default. On the cluster, the logs sit on HDFS under <code>/var/log/spark/apps/</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -ls</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /var/log/spark/apps/</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -get</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /var/log/spark/apps/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span></span></code></pre></div><p>If your cluster sets <code>spark.eventLog.dir</code> to an S3 path, download the file from there instead (for example with <code>aws s3 cp</code>). AWS&#39;s <a href="https://aws.github.io/aws-emr-best-practices/docs/benchmarks/Analyzing/retrieve_event_logs/" target="_blank" rel="noreferrer">EMR best-practices guide</a> covers both.</p><h2 id="aws-glue" tabindex="-1">AWS Glue <a class="header-anchor" href="#aws-glue" aria-label="Permalink to &quot;AWS Glue&quot;">​</a></h2><p>Glue writes event logs to S3 when the job has the Spark UI turned on. In the job parameters:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>--enable-spark-ui true</span></span>
+<span class="line"><span>--spark-event-logs-path s3://my-bucket/spark-events/</span></span>
+<span class="line"><span>--enable-spark-ui-legacy-path true</span></span></code></pre></div><p>Glue has two log formats: Standard, for its own console, and Legacy, which AWS documents for viewing on a Spark History Server. Ask for Legacy, as the last parameter above does, or pick it in the console under <strong>Spark UI logging and monitoring configuration</strong>.</p><p>After the run, download the file, or the rolling-log directory when the job uses <code>spark.eventLog.rolling.enabled</code>, from that S3 path. See <a href="https://docs.aws.amazon.com/glue/latest/dg/monitor-spark-ui-jobs.html" target="_blank" rel="noreferrer">Enabling the Apache Spark web UI for AWS Glue jobs</a>.</p><h2 id="databricks" tabindex="-1">Databricks <a class="header-anchor" href="#databricks" aria-label="Permalink to &quot;Databricks&quot;">​</a></h2><p>Set up compute log delivery on the cluster or job compute before it runs: <strong>Advanced &gt; Logging</strong>, then pick a Unity Catalog volume, S3 or DBFS path. Databricks delivers driver, worker and event logs into a subfolder of that path named after the cluster ID, and keeps delivering until the compute shuts down. Copy the event log from there. See <a href="https://docs.databricks.com/aws/en/compute/configure#compute-log-delivery" target="_blank" rel="noreferrer">compute log delivery</a>.</p><h2 id="dataproc" tabindex="-1">Dataproc <a class="header-anchor" href="#dataproc" aria-label="Permalink to &quot;Dataproc&quot;">​</a></h2><p>Google Cloud Dataproc (also called Managed Service for Apache Spark) saves Spark job history to the cluster&#39;s temp bucket, in its <code>/spark-job-history</code> directory, unless the cluster sets <code>spark.eventLog.dir</code> somewhere else. Download the file with <code>gcloud storage cp</code>. See <a href="https://cloud.google.com/dataproc/docs/concepts/jobs/history-server" target="_blank" rel="noreferrer">Persistent History Server</a>.</p>`,15)])])}const u=a(o,[["render",n]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.lean.js

@@ -0,0 +1 @@
+import{_ as a,o as t,c as s,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Managed platforms","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval/managed-platforms.md","filePath":"user-guide/alternative-log-retrieval/managed-platforms.md"}'),o={name:"user-guide/alternative-log-retrieval/managed-platforms.md"};function n(i,e,l,p,d,h){return t(),s("div",null,[...e[0]||(e[0]=[r("",15)])])}const u=a(o,[["render",n]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.js

@@ -0,0 +1 @@
+import{_ as s,o as a,c as t,a5 as i}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Behind an SSH bastion","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval/ssh-bastion.md","filePath":"user-guide/alternative-log-retrieval/ssh-bastion.md"}'),o={name:"user-guide/alternative-log-retrieval/ssh-bastion.md"};function r(h,e,n,l,p,d){return a(),t("div",null,[...e[0]||(e[0]=[i('<h1 id="behind-an-ssh-bastion" tabindex="-1">Behind an SSH bastion <a class="header-anchor" href="#behind-an-ssh-bastion" aria-label="Permalink to &quot;Behind an SSH bastion&quot;">​</a></h1><p>How to fetch a History Server log when the server is only reachable through an SSH bastion.</p><p>The most common hard case is a Spark History Server (SHS) reachable only through an SSH bastion or jump host (the recipe below is the same regardless of what&#39;s issuing the SSH session), with the event logs themselves living on Kerberized HDFS and no direct HTTP path from your machine to SHS.</p><h2 id="why-shs-base-url-won-t-work-here" tabindex="-1">Why <code>--shs-base-url</code> won&#39;t work here <a class="header-anchor" href="#why-shs-base-url-won-t-work-here" aria-label="Permalink to &quot;Why `--shs-base-url` won&#39;t work here&quot;">​</a></h2><p>The CLI&#39;s <code>--shs-base-url</code>/<code>--app-id</code> flags and the MCP tools&#39; <code>{ shsBaseUrl, appId }</code> source both make a direct HTTP request from wherever SparkForensics runs to the History Server&#39;s REST API. There&#39;s no <code>--via-ssh</code> flag. If the bastion can reach the History Server&#39;s HTTP port, forward it (<code>ssh -L 18080:&lt;shs-host&gt;:18080 &lt;bastion&gt;</code>) and use <code>--shs-base-url http://127.0.0.1:18080</code>. SparkForensics sends no credentials, so a History Server that requires Kerberos/SPNEGO sign-in can&#39;t be fetched this way: use the recipe below.</p><p>Running in Airflow? The <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a> can fetch the log from a History Server behind an SSH tunnel, or run the analysis on an SSH host that already sees the logs.</p><h2 id="the-recipe" tabindex="-1">The recipe <a class="header-anchor" href="#the-recipe" aria-label="Permalink to &quot;The recipe&quot;">​</a></h2><ol><li><p>SSH into the edge node (through the bastion, or whatever gets you there).</p></li><li><p>Pull the event log off HDFS onto local disk on the edge node:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -get</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /path/to/spark-events/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /tmp/application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Copy that file back to your own machine through the same bastion, e.g. with <code>scp</code> or <code>sftp</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">scp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> edge-node:/tmp/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ./application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Load the local copy as described in <a href="./../alternative-log-retrieval.html#loading-the-file">Loading the file</a>.</p></li></ol>',8)])])}const b=s(o,[["render",r]]);export{g as __pageData,b as default};

package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.lean.js

@@ -0,0 +1 @@
+import{_ as s,o as a,c as t,a5 as i}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Behind an SSH bastion","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval/ssh-bastion.md","filePath":"user-guide/alternative-log-retrieval/ssh-bastion.md"}'),o={name:"user-guide/alternative-log-retrieval/ssh-bastion.md"};function r(h,e,n,l,p,d){return a(),t("div",null,[...e[0]||(e[0]=[i("",8)])])}const b=s(o,[["render",r]]);export{g as __pageData,b as default};

package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Getting started","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started.md","filePath":"user-guide/getting-started.md"}'),n={name:"user-guide/getting-started.md"};function s(i,e,l,d,p,c){return o(),a("div",null,[...e[0]||(e[0]=[r('<h1 id="getting-started" tabindex="-1">Getting started <a class="header-anchor" href="#getting-started" aria-label="Permalink to &quot;Getting started&quot;">​</a></h1><p>SparkForensics reads a Spark History Server event log and turns it into a dashboard of flagged bottlenecks. No account needed.</p><h2 id="local-server-mode" tabindex="-1">Run it locally <a class="header-anchor" href="#local-server-mode" aria-label="Permalink to &quot;Run it locally {#local-server-mode}&quot;">​</a></h2><p>The recommended way to run SparkForensics on your machine:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-server</span></span></code></pre></div><p>Open <code>http://127.0.0.1:4173</code> and drop a log in. This is <strong>local-server mode</strong>: it also fetches runs from a Spark History Server on your behalf. Port and environment-variable overrides, and what changes for a static deploy, are in the <a href="https://github.com/shuffle-works/sparkforensics#deploy-modes" target="_blank" rel="noreferrer">project README</a>.</p><p>The <a href="https://shuffle-works.github.io/sparkforensics/" target="_blank" rel="noreferrer">hosted demo</a> runs the same dashboard but can&#39;t fetch from a History Server. To work on SparkForensics itself, see <a href="./../contributor-guide/development-setup.html">Development setup</a>.</p><h2 id="load-a-run" tabindex="-1">Load a run <a class="header-anchor" href="#load-a-run" aria-label="Permalink to &quot;Load a run&quot;">​</a></h2><p>Drop a log file onto the landing page, or click <strong>Choose file</strong> to pick one. Parsing runs in a background worker, off the browser&#39;s main thread, so a multi-hundred-megabyte event stream doesn&#39;t freeze the tab. While it runs, the progress screen shows how far along it is and roughly how long is left; <strong>Cancel</strong> returns to the landing page.</p><p>No log of your own yet? Click <strong>Try a sample run</strong> on the landing page to load a bundled example run and see a populated dashboard right away. A line above its verdict says it is the sample, with <strong>Load my event log</strong> and a link to where to find one, so you can switch to your own run from there. <strong>Where do I find my event log?</strong>, under the buttons, is a short guide to turning event logging on and downloading a log from a History Server.</p><p>The app takes a newline-delimited JSON event log (one JSON event per line, the format Spark writes to <code>spark.eventLog.dir</code>), either plain or gzip/Zstandard/LZ4/Snappy-compressed. It also takes the zip a Spark History Server hands back, from the Spark UI&#39;s download link or from <code>GET /api/v1/applications/&lt;appId&gt;/logs</code>, as-is: drop the <code>.zip</code> and the app unwraps the log inside it, or reassembles the parts of a rolling log. The zip must hold one attempt: for an application that ran more than once, download <code>/api/v1/applications/&lt;appId&gt;/&lt;attemptId&gt;/logs</code> instead. Spark&#39;s <code>lzf</code> codec is not supported: set <code>spark.eventLog.compression.codec</code> to <code>zstd</code>, <code>lz4</code> or <code>snappy</code>.</p><p>Click <strong>Other sources</strong> on the landing page for two more ways in:</p><ul><li><strong>Choose rolling-log folder</strong>, for an <code>eventlog_v2_*</code> rolling directory. Drop the folder or point the picker at it; the app reassembles its parts in order before parsing.</li><li><strong>Fetch from Spark History Server</strong>, to pull an application straight from a reachable History Server. This needs <strong>local-server mode</strong> (see <a href="#local-server-mode">Run it locally</a>) and an application ID in one of the forms Spark itself uses: <code>application_&lt;timestamp&gt;_&lt;id&gt;</code>, <code>local-&lt;timestamp&gt;</code>, <code>app-&lt;id&gt;</code>, <code>spark-&lt;id&gt;</code>, or <code>driver-&lt;id&gt;</code>. If the fetch fails, the panel names the problem (server unreachable, application not found, local server not running, and so on) and suggests what to try next.</li></ul><p>Can&#39;t reach the History Server directly (it&#39;s only reachable through an SSH bastion)? See <a href="./alternative-log-retrieval/ssh-bastion.html#behind-an-ssh-bastion">Behind an SSH bastion</a>.</p><p>In Chromium-based browsers, single files you open stay listed under <strong>Recent files</strong> on the landing page. Rolling-log folders, History Server fetches and the sample run are not listed.</p>',15)])])}const u=t(n,[["render",s]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.lean.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Getting started","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started.md","filePath":"user-guide/getting-started.md"}'),n={name:"user-guide/getting-started.md"};function s(i,e,l,d,p,c){return o(),a("div",null,[...e[0]||(e[0]=[r("",15)])])}const u=t(n,[["render",s]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.DpnfKz3x.js

@@ -0,0 +1,22 @@
+import{_ as t,o as a,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"CI and automation","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/ci-and-automation.md","filePath":"user-guide/getting-started/ci-and-automation.md"}'),d={name:"user-guide/getting-started/ci-and-automation.md"};function i(n,e,c,r,l,h){return a(),o("div",null,[...e[0]||(e[0]=[s(`<h1 id="ci-and-automation" tabindex="-1">CI and automation <a class="header-anchor" href="#ci-and-automation" aria-label="Permalink to &quot;CI and automation&quot;">​</a></h1><p>How to run SparkForensics from a pipeline or script with the <code>analyze</code> CLI: its output formats, the extra data in the JSON report, and gating on several budgets and candidates.</p><p>In a pipeline or a script, use the <code>analyze</code> CLI instead of the browser dashboard. It parses the same event logs and exits non-zero when a run crosses a threshold you set, so it can gate a build:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -p</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-cli</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-analyze</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt; [--max-runtime </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ms]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-skew </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ratio]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  [--max-spill </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">gb]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-failed-task-rate </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--min-efficiency </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  [--out </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">path]</span></span></code></pre></div><p>Output is JSON by default; <code>--format md</code> writes the Markdown report instead. Budget violations and inconclusive budgets print to stderr as <code>[violation] ...</code> and <code>[inconclusive] ...</code> lines. The exit code is 0 when every budget passes, 1 when one is violated, 3 when none is violated but one is inconclusive, 2 for a usage error (bad flags or arguments, an invalid <code>--thresholds</code> or <code>--budgets</code> file), 4 when the candidate log can&#39;t be read or parsed or its History Server fetch fails, 5 when the <code>--baseline</code> log can&#39;t be read or parsed, and 6 for an internal error, including a failed <code>--export-html</code>. When several of these apply, the worst wins in this order: 6, 5, 4, 1, 3, 0. A usage error exits 2 before any log is read.</p><p><code>--min-efficiency</code> checks busy core time, the share of executor core time that ran tasks (100 minus the dashboard&#39;s Unused core time). It is not the dashboard&#39;s Efficiency tile, which is the share of wall-clock with a stage running, so the two can differ widely on the same run.</p><p>The command ships in the <code>sparkforensics-cli</code> package. To install it once: <code>npm i -g sparkforensics-cli</code>, then run <code>sparkforensics-analyze</code> directly. <code>npx sparkforensics-analyze &lt;file&gt;</code> is a shorter way to run the same command.</p><p>Point it at a single event-log file, or at one run&#39;s <code>eventlog_v2_*</code> rolling-log directory; any other directory is rejected. Output goes to stdout; pass <code>--out &lt;path&gt;</code> to write it to that file instead.</p><p>Pass <code>--export-html &lt;dir&gt;</code> to write a self-contained HTML dashboard for the run into <code>&lt;dir&gt;</code> (which must not already exist or must be empty). Open <code>&lt;dir&gt;/index.html</code> directly in a browser over <code>file://</code>, with no server, to get the same interactive dashboard offline, without the docs links. This makes it easy to archive or share a run. <code>--redact</code> applies to the exported report too.</p><h2 id="machine-readable-fixes-and-costs" tabindex="-1">Machine-readable fixes and costs <a class="header-anchor" href="#machine-readable-fixes-and-costs" aria-label="Permalink to &quot;Machine-readable fixes and costs&quot;">​</a></h2><p>Each row of the JSON report&#39;s <code>findings</code> array carries two fields for a script or tuning loop that acts on the output without reading prose.</p><p><code>remediation</code> is an array of the property changes the row&#39;s <code>recommendation</code> names, in structured form. It is empty when the recommendation names no Spark property.</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;remediation&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;kind&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;conf&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;key&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;spark.sql.shuffle.partitions&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;direction&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;increase&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;suggested&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">800</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span></code></pre></div><ul><li><code>kind</code> is always <code>&quot;conf&quot;</code>: a Spark property.</li><li><code>direction</code> is <code>&quot;increase&quot;</code> or <code>&quot;decrease&quot;</code> (move the current value that way) or <code>&quot;set&quot;</code> (take the <code>suggested</code> value, for a switch or a class name).</li><li><code>suggested</code> is the value the detector computed, or <code>null</code> when it computes none. Counts are numbers, switches are booleans, sizes carry a Spark unit suffix (<code>&quot;1024m&quot;</code>).</li></ul><p><code>impactEstimate.coreTimeMs</code> is the busy core time the fix removes: the executor task time, in core-milliseconds, next to the <code>wallClock</code> range (elapsed time). It is a <code>{ &quot;low&quot;: ..., &quot;high&quot;: ... }</code> range with <code>low</code> equal to <code>high</code>, or <code>null</code> when the detector measures no such figure: <code>null</code> means unknown, never zero. Only figures read from the log count: GC time, the executor time of retried and discarded speculative attempts, and for skew and straggler findings the task time the fix removes from the slow tasks. These are set even when the log has no executor core data. A finding with only a wall-clock claim has <code>coreTimeMs: null</code>: its elapsed time is not converted to core time. Task run time is used, not <code>executorCpuTime</code>, so Python worker CPU is not missed.</p><p>Findings whose only figure is bytes or memory-time have <code>coreTimeMs: null</code>, as do findings whose core figure rests on an assumed constant: <code>coreLocality</code>&#39;s per-task fetch penalty and the executor-hours or job-hours of <code>autoscalingChurn</code> and <code>jobFailureRate</code>. Their <code>rawWaste</code> is unchanged.</p><p><code>coreTimeMs</code> never includes idle capacity. A finding whose waste is allocated cores that ran no task (<code>utilization</code>, <code>stageShape</code>&#39;s low parallelism and task/stage skew rows) has <code>coreTimeMs: null</code> and keeps its figure in <code>impactEstimate.rawWaste</code> with <code>&quot;idle&quot;: true</code>; every surface labels it &quot;of idle core capacity&quot;, not &quot;of core time&quot;. A stage&#39;s slow tail is counted once: when <code>skew</code> and <code>straggler</code> both flag the same stage, <code>skew</code> carries the removed task time and <code>straggler</code> has <code>null</code>.</p><p>When the logged <code>spark.sql.shuffle.partitions</code> is already at or above the count a low-parallelism shuffle stage needs, the property is not what limits that stage: the recommendation points at the stage&#39;s own partitioning (<code>repartition(n)</code> or RDD parallelism) and <code>remediation</code> is empty.</p><p>A <code>remediation</code> that sets a property to a fixed value (for example <code>spark.sql.adaptive.skewJoin.enabled</code>, <code>spark.speculation</code> or <code>spark.dynamicAllocation.enabled</code>) is left out when the run&#39;s logged conf already has that value; booleans compare case-insensitively. The recommendation then stops naming that property and points at the remedy left (for example &quot;AQE skew-join handling is already on, so salt the key or repartition on a better key&quot;), so the text and <code>remediation</code> never disagree. The dashboard&#39;s one-line fix for a group of such findings follows the same logged conf. Only properties the event log records count: Spark&#39;s unlogged version defaults (such as skew-join handling being on by default with AQE in Spark 3.2+) are not modeled, so such a run can still get the suggestion.</p><p>Two logged settings change which fix is offered. Skew-join handling counts as already on only when <code>spark.sql.adaptive.enabled</code> is not logged <code>false</code>; with AQE logged off, the skew findings suggest setting <code>spark.sql.adaptive.enabled</code> to <code>true</code> instead. When <code>spark.sql.autoBroadcastJoinThreshold</code> is logged <code>-1</code> (auto-broadcast disabled), an over-broadcast finding has an empty <code>remediation</code> and points at removing the <code>broadcast()</code> hint.</p><p>The CLI also supports fetching a run directly from a reachable Spark History Server (<code>--shs-base-url</code>/<code>--app-id</code>/<code>--attempt-id</code>) instead of a local file, comparing a candidate run against a baseline with regression gating (<code>--baseline</code>/<code>--max-regression-pct</code>/<code>--regression-metric</code>/ <code>--fail-on-introduced</code>; the baseline is a local file or rolling-log directory, no History Server), several regression budgets and several candidates in one call (<code>--regression-budget</code>/<code>--budgets</code>, see <a href="#several-budgets-and-several-candidates">Several budgets and several candidates</a>), rewriting run-specific text before stages are paired (<code>--normalize-path</code>, see <a href="#pairing-stages-across-runs">Pairing stages across runs</a>), redacting the app id, the app name and any host/IP tokens before sharing output (<code>--redact</code>), and narrowing the findings to certain impact bands, types, or a stage (<code>--impact</code>/<code>--type</code>/<code>--stage</code>), and tuning detector thresholds from a file (<code>--thresholds</code>, below). Run it with <code>--help</code> for the full flag list.</p><p>Same caveat as above: if the History Server is only reachable through an SSH bastion, <code>--shs-base-url</code> can&#39;t reach it either: see <a href="./../alternative-log-retrieval/ssh-bastion.html#behind-an-ssh-bastion">Behind an SSH bastion</a>.</p><p>Running in Airflow instead of a plain CI pipeline? See <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a>, an Airflow operator that wraps the CLI and acts on the result after each Spark job, so you don&#39;t have to wire up the call yourself.</p><p>Want an AI assistant to diagnose a run directly, without the dashboard or a CI gate? See <a href="./../mcp-tools.html">MCP tools reference</a>.</p><h2 id="write-targets" tabindex="-1">Write targets <a class="header-anchor" href="#write-targets" aria-label="Permalink to &quot;Write targets {#write-targets}&quot;">​</a></h2><p>The JSON report has a top-level <code>writeTargets</code> object listing every SQL write in the run, so a script can check where a job wrote without reading the plan. It has no Markdown counterpart. The MCP <code>diagnose_run</code> tool returns it too.</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;writes&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;sqlExecutionId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">3</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;nodeId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;e3:n0&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;command&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;InsertIntoHadoopFsRelationCommand&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;recognized&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;kind&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;target&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;hdfs://nn/sandbox/out/t1&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;outputRows&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">5000000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;raw&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Execute InsertIntoHadoopFsRelationCommand hdfs://nn/sandbox/out/t1, false, Parquet, ...&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;executionsWithoutPlan&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;sqlExecutionId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">4</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;reason&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;noPlan&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;skippedLines&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><ul><li><p><code>writes</code> holds one row per write node, ordered by SQL execution id and then by position in the plan. A <code>DeltaTable</code> API merge has no write node: it is one extra row per merge, see <a href="#delta-writes">Delta writes</a>. <code>command</code> is the plan node&#39;s name without the <code>Execute </code> prefix or an <code>Exec</code> suffix, <code>nodeId</code> is the plan node&#39;s id, and <code>raw</code> is the node&#39;s full plan string as the log recorded it.</p></li><li><p><code>kind</code> says how to read <code>target</code>:</p><ul><li><code>path</code> is a filesystem location.</li><li><code>table</code> is a table name whose catalog the command states or implies: a Hive or V1 command&#39;s <code>database.table</code>, a DataSource V2 name with a catalog part (<code>catalog.namespace.table</code>), or a Delta command&#39;s table as <code>database.table</code> (a table in the session catalog) or <code>catalog.database.table</code>.</li><li><code>unqualifiedTable</code> is a DataSource V2 name with no catalog part, such as <code>default.events</code> or <code>events</code>. The log does not say which catalog it lives in, so a check on its database part alone can match the wrong catalog.</li><li><code>jdbcTable</code> is a <code>dbtable</code> of a JDBC write. It names a table in an external database, not in a Spark catalog, and the database itself (the JDBC url) is not reported.</li></ul><p>Every target is copied verbatim from the plan string: nothing is resolved, so a relative path, an unexpanded <code>\${var}</code> placeholder or a bare table name appears exactly as the log states it. Backticks around table name parts are dropped.</p></li><li><p><code>target</code> and <code>kind</code> are <code>null</code> when the target cannot be determined. That covers a node whose plan string has no target (Spark omits it for some commands, and Delta command nodes carry none on real logs, which <a href="#delta-writes">Delta writes</a> covers), a redacted option value, a plan string that names more than one candidate target, and a plan string Spark cut short. A target is reported only when its whole text is followed by a delimiter in the plan string: a path cut by <code>spark.sql.maxMetadataStringLength</code> or a name with a <code>...</code> marker in it is <code>null</code>, never a partial path. A trailing <code>... N more fields</code> after the target, such as the column list of <code>InsertIntoHadoopFsRelationCommand</code>, does not affect the target before it. Treat a <code>null</code> target as unknown, not as inside or outside any location.</p></li><li><p><code>outputRows</code> is the node&#39;s own <code>number of output rows</code> SQL metric. It is <code>null</code> when the log has no such metric for that node, which is the case for the DataSource V2 and Delta write nodes that report other metrics.</p></li><li><p><code>recognized</code> is <code>true</code> for the commands below and <code>false</code> for a write-like node that is not in that list. An unrecognized write always has <code>target</code> and <code>kind</code> <code>null</code>, and <code>raw</code> is the only information about it.</p></li><li><p><code>executionsWithoutPlan</code> lists the SQL executions whose writes the report cannot see, each with a <code>reason</code>. <code>noPlan</code> means the execution started but the log has no plan for it, such as an execution still running when the log was cut off (a plan is read when its execution ends). <code>unreadableStart</code> means the execution&#39;s start event was skipped because it could not be read, for example a plan nested deeper than 500 nodes or a start line cut off at the end of the log. A write in one of these executions is not in <code>writes</code>, so a non-empty list means the report may be incomplete.</p></li><li><p><code>skippedLines</code> is the number of log lines the parser could not read. A line cut off before its execution id is not in <code>executionsWithoutPlan</code>, so a non-zero count means a write may be missing even when that list is empty. It is <code>null</code> when the count is unknown.</p></li></ul><p>Recognized commands and where their target comes from:</p><table tabindex="0"><thead><tr><th>Command</th><th>Target</th></tr></thead><tbody><tr><td><code>InsertIntoHadoopFsRelationCommand</code></td><td><code>path</code>: the first argument, including for an <code>INSERT OVERWRITE ... PARTITION</code> that prints a static-partition map after it</td></tr><tr><td><code>InsertIntoHiveTable</code>, <code>CreateDataSourceTableAsSelectCommand</code>, <code>CreateHiveTableAsSelectCommand</code>, <code>OptimizedCreateHiveTableAsSelectCommand</code></td><td><code>table</code>: the first argument</td></tr><tr><td><code>SaveIntoDataSourceCommand</code></td><td><code>path</code> from the <code>path</code> option, or <code>jdbcTable</code> from the JDBC <code>dbtable</code>/<code>table</code> option</td></tr><tr><td><code>AppendData</code>, <code>OverwriteByExpression</code>, <code>OverwritePartitionsDynamic</code>, <code>ReplaceData</code>, <code>WriteDelta</code>, <code>WriteToDataSourceV2</code>, <code>AppendDataExecV1</code>, <code>OverwriteByExpressionExecV1</code></td><td><code>table</code> or <code>unqualifiedTable</code>: the <code>table=</code> of the connector&#39;s write object, for example Iceberg&#39;s <code>IcebergWrite(table=..., ...)</code>. For a Delta table, see <a href="#delta-writes">Delta writes</a></td></tr><tr><td><code>CreateTableAsSelect</code>, <code>AtomicCreateTableAsSelect</code>, <code>ReplaceTableAsSelect</code>, <code>AtomicReplaceTableAsSelect</code></td><td><code>table</code> or <code>unqualifiedTable</code>: the identifier after the catalog object, which does not name the catalog</td></tr><tr><td><code>WriteIntoDelta</code>, <code>WriteIntoDeltaCommand</code>, <code>UpdateCommand</code>, <code>DeleteCommand</code>, <code>MergeIntoCommand</code></td><td><code>path</code>: the first argument when it is a <code>delta.</code> path table (never for <code>MergeIntoCommand</code>, whose plan string also prints the source); otherwise as in <a href="#delta-writes">Delta writes</a></td></tr><tr><td><code>CreateDeltaTableCommand</code>, <code>OptimizeTableCommand</code>, <code>RestoreTableCommand</code>, <code>DeltaReorgTableCommand</code></td><td><code>path</code>: the first argument when it is a <code>delta.</code> path table, otherwise <code>null</code></td></tr><tr><td><code>DeltaMerge</code></td><td>not a plan node: one row per <code>DeltaTable</code> API merge, see <a href="#delta-writes">Delta writes</a></td></tr></tbody></table><p>A plan node not in the table is a write when its operator name (the first word of the node name after any leading <code>Execute</code>, so not the relation or table a scan prints after it), split into CamelCase words, contains <code>Write</code>, <code>Insert</code>, <code>Save</code>, <code>Overwrite</code>, <code>Append</code>, <code>Merge</code>, <code>Update</code>, <code>Delete</code>, <code>Truncate</code>, <code>Replace</code>, <code>Drop</code>, <code>Load</code>, <code>Vacuum</code>, <code>Convert</code>, <code>Clone</code>, <code>Restore</code>, <code>Optimize</code>, <code>Alter</code>, <code>Reorg</code>, <code>Rename</code> or <code>Call</code>, or contains <code>TableAsSelect</code>, <code>AddPartition</code>, <code>DropPartition</code>, <code>RenamePartition</code> or <code>RecoverPartitions</code>. That covers table DDL and <code>ADD PARTITION ... LOCATION</code>, and an Iceberg procedure call (<code>Call</code>), whose target is a procedure argument and not a path, so it is reported with a <code>null</code> target. A name containing <code>Join</code> (<code>SortMergeJoin</code>) and these operators that share a word with a write but write nothing are not: <code>WriteFiles</code> (the child of a write command), <code>AppendColumns</code>, <code>AppendColumnsWithObject</code>, <code>MergeRows</code>, <code>StateStoreSave</code>, <code>StateStoreRestore</code>, <code>SessionWindowStateStoreSave</code>, <code>SessionWindowStateStoreRestore</code> and <code>UpdateEventTimeWatermarkColumn</code>. The classification leans toward reporting too much: a node that matches by name is listed with a <code>null</code> target rather than dropped.</p><h3 id="delta-writes" tabindex="-1">Delta writes <a class="header-anchor" href="#delta-writes" aria-label="Permalink to &quot;Delta writes {#delta-writes}&quot;">​</a></h3><p>On a real log a Delta command node (<code>MergeIntoCommand</code>, <code>UpdateCommand</code>, <code>DeleteCommand</code>, <code>WriteIntoDelta</code>, <code>SaveIntoDataSourceCommand</code>) is a bare name: its plan string has no target. The target is read from up to two other places, in this order, and is <code>null</code> when neither names exactly one target.</p><ol><li>The command&#39;s <code>Arguments:</code> line in Spark&#39;s physical plan description. The parser drops the rest of that text to save memory, but keeps this one line for these commands. A quoted table name that appears once on the line is the target, as <code>kind: &quot;table&quot;</code> (<code>spark_catalog</code> is dropped, so it reads <code>database.table</code>). A <code>SaveIntoDataSourceCommand</code> is read this way only when the line shows a Delta source, and its target is the <code>path</code> option. A description that spans two read chunks is not kept for a <code>SaveIntoDataSourceCommand</code>, which prints its whole query plan under it; its target is then <code>null</code>.</li><li>For <code>MergeIntoCommand</code>, <code>UpdateCommand</code> and <code>DeleteCommand</code> only, the <code>_delta_log</code> path in the plans of the executions that share the command&#39;s root execution, as <code>kind: &quot;path&quot;</code>. Exactly one distinct path must appear. It applies only when the command is its own root execution, and a source that is itself a Delta table adds a second path and leaves the target <code>null</code>. The write commands (<code>WriteIntoDelta</code>, <code>WriteIntoDeltaCommand</code>, <code>SaveIntoDataSourceCommand</code>) never take this step, because a write may read another Delta table as its source: with no table or <code>path</code> option on the arguments line, their target is <code>null</code>.</li></ol><p>An append or overwrite of an existing Delta table (<code>saveAsTable</code> in <code>append</code> or <code>overwrite</code> mode on an existing table, <code>INSERT INTO</code>, <code>INSERT OVERWRITE</code>) prints the table object as its first argument, and that names the table: <code>kind: &quot;table&quot;</code> as <code>database.table</code> (<code>catalog.database.table</code> outside the session catalog), or <code>kind: &quot;path&quot;</code> for a path-based table. A <code>CREATE TABLE ... AS SELECT</code>, <code>CREATE OR REPLACE TABLE</code> or <code>saveAsTable</code> that creates or replaces a table runs two executions under one root: the create or replace node, which names the table as <code>unqualifiedTable</code> because Delta&#39;s catalog object does not state its catalog name, and an <code>AppendDataExecV1</code> whose staged table has no name. That append takes the table its root&#39;s create or replace node names, and is <code>null</code> when the root has no single such node.</p><p>A merge made with <code>DeltaTable.merge(...).execute()</code> normally runs a <code>MergeIntoCommand</code> too (its description is <code>toDataset$ at DeltaMergeBuilder</code>), and is reported like a SQL MERGE, including two merges run from threads of one session, whose sub-queries are told apart by their root execution. Some logs have no such command execution for an API merge. Its queries are then separate root executions, and the only things tying them together are a <code>MERGE operation</code> description and adjacent execution ids. Each run of consecutive ids with such a description is reported as one write with the synthetic command <code>DeltaMerge</code>, <code>kind: &quot;path&quot;</code> and the table path from the <code>_delta_log</code> of the plans in the run. The log holds no table name for these merges. The run must hold a write phase (a description containing <code>writing</code> or <code>rewriting</code>), because a run that only scanned may have read the merge&#39;s source. The target is <code>null</code> when the run has no write phase, names zero or several paths, has a plan or start time missing, or has start times that overlap another run&#39;s (merges on concurrent threads); <code>raw</code> says which. Two merges whose executions have consecutive ids are one run and name two paths, so they get no target. A description a user sets that starts with <code>MERGE operation</code> is indistinguishable from Delta&#39;s and is reported the same way. <code>raw</code> reads <code>Delta MERGE operation, executions 10-17</code>.</p><p>Only SQL writes are covered. Writes made outside Spark SQL, such as an RDD <code>saveAsTextFile</code> or a direct filesystem call from the driver, do not appear in the event log and are not in <code>writeTargets</code>.</p><h2 id="metrics-block" tabindex="-1">Metrics block <a class="header-anchor" href="#metrics-block" aria-label="Permalink to &quot;Metrics block&quot;">​</a></h2><p>The CLI&#39;s JSON output carries a <code>metrics</code> block next to the report, for scripts that act on a run without reading the report. With <code>--baseline</code> it is inside <code>candidate</code>. The MCP <code>diagnose_run</code> tool returns the same block. It has its own <code>schemaVersion</code>, separate from the report&#39;s. A figure the log cannot provide is <code>null</code>, never <code>0</code>: a log from a Spark version that records no CPU time has <code>executorCpuTimeMs: null</code>, and a log cut off before any executor joined has null allocation. The time, data and task figures, run-level and per row, include the work of every attempt of a resubmitted stage, failed attempts too, and the tasks of a failed attempt that end after it (killed or still running when it failed); skew describes the latest attempt.</p><table tabindex="0"><thead><tr><th>Field</th><th>Meaning</th></tr></thead><tbody><tr><td><code>runComplete</code></td><td><code>true</code> when the log has an application-end record. <code>false</code> means the log was cut off and every total covers only what it recorded.</td></tr><tr><td><code>time.wallClockMs</code></td><td>Application start to end; null without both.</td></tr><tr><td><code>time.executorCpuTimeMs</code></td><td>Summed CPU time of every task attempt, including failed attempts, retries and speculative copies that lost. Null when no task recorded any. Misses Python worker CPU, see <code>python</code>.</td></tr><tr><td><code>time.executorRunTimeMs</code>, <code>time.gcTimeMs</code></td><td>Summed run time and JVM GC time of every task attempt, counted like the CPU time.</td></tr><tr><td><code>data.memorySpillBytes</code>, <code>data.diskSpillBytes</code></td><td>Summed spill.</td></tr><tr><td><code>data.shuffleReadBytes</code>, <code>data.shuffleWriteBytes</code></td><td>Summed shuffle bytes, local plus remote on the read side.</td></tr><tr><td><code>data.inputBytes</code>, <code>data.outputBytes</code>, <code>data.outputRows</code></td><td>Summed input, output and rows written. <code>outputRows</code> is null when no task reported rows.</td></tr><tr><td><code>data.peakExecutionMemoryBytes</code></td><td>The largest per-task peak execution memory. Null when every task reports 0.</td></tr><tr><td><code>shape.taskCount</code>, <code>shape.stageCount</code></td><td>Distinct task records and stages.</td></tr><tr><td><code>shape.failedStageAttempts</code></td><td>Stage attempts that ended with a failure reason, including an attempt Spark then resubmitted (a fetch failure, for example).</td></tr><tr><td><code>shape.retriedStages</code></td><td>Stages submitted more than once.</td></tr><tr><td><code>shape.failedTasks</code>, <code>shape.retriedTasks</code></td><td>Task-level counts: tasks whose final attempt failed, and task attempts superseded by a retry.</td></tr><tr><td><code>shape.maxSkew</code></td><td>The largest stage skew ratio, the figure <code>--max-skew</code> checks (P95 over median, or max over median for a stage with few tasks; <code>skew.minTasksForP95</code> from <code>--thresholds</code> applies).</td></tr><tr><td><code>allocation.coreHours</code>, <code>allocation.memoryGbHours</code></td><td>See <a href="#allocation">Allocation</a>.</td></tr><tr><td><code>python.shareOfTaskRunTime</code></td><td>See <a href="#python-share">Python share</a>.</td></tr><tr><td><code>stages</code></td><td>The per-stage rows.</td></tr></tbody></table><p><code>stages</code> holds one row per stage fingerprint, keyed by the same fingerprint the run comparison matches stages on (the normalized stage name plus the stage&#39;s plan nodes), so the same stage has the same key in a baseline and a candidate run. A stage repeated in a loop shares one key: its row sums the repeats and lists them in <code>stageIds</code>. A row carries <code>durationMs</code>, <code>executorCpuTimeMs</code>, <code>executorRunTimeMs</code>, <code>gcTimeMs</code>, the spill, shuffle, input and output figures, <code>outputRows</code>, <code>peakExecutionMemoryBytes</code>, <code>taskCount</code>, <code>failedTasks</code>, <code>retriedTasks</code> and <code>skew</code> (the largest skew ratio among the repeats) as above, plus <code>failed</code> (an attempt of the stage failed), <code>retried</code> (the stage was submitted more than once) and <code>python</code> flags. The stage-attempt figures, run-level and per row, are null for a stage record that carries no attempt count. With <code>--redact</code> the <code>metrics</code> and <code>effectiveConf</code> blocks are built from the redacted run the report uses, so they carry the same <code>host-N</code> and <code>app-N</code> pseudonyms: the keys come from redacted stage names.</p><h3 id="allocation" tabindex="-1">Allocation <a class="header-anchor" href="#allocation" aria-label="Permalink to &quot;Allocation&quot;">​</a></h3><p><code>allocation.coreHours</code> is the sum over executors of cores times hours alive. <code>allocation.memoryGbHours</code> is the sum of memory times hours alive, in GiB (1024 MiB). An executor is alive from its executor-added event to its first later executor-removed event. One with no removal event closes at the application end when the log has one. In a cut-off log (<code>runComplete</code> false) it closes at the last timestamp the log records (the latest application, stage or executor event), so the figure is a lower bound.</p><p>Cores per executor are the executor-added event&#39;s total cores. Under dynamic allocation or YARN defaults, where <code>spark.executor.cores</code> is not set, that is the core count the cluster manager actually granted; when the event carries none, <code>spark.executor.cores</code> is used. If neither exists the core-hours are null.</p><p>Memory per executor is the container size Spark requests: <code>spark.executor.memory</code> (1g when unset), plus the overhead, plus <code>spark.memory.offHeap.size</code> when <code>spark.memory.offHeap.enabled</code> is <code>true</code>, plus <code>spark.executor.pyspark.memory</code>. The overhead is <code>spark.executor.memoryOverhead</code>, else the legacy <code>spark.yarn.executor.memoryOverhead</code>, else the larger of 384 MiB and <code>spark.executor.memoryOverheadFactor</code> (default 0.1) times the executor memory. Memory GB-hours are null when the log records no Spark properties at all, or a memory property cannot be read. Driver resources are not included.</p><h3 id="python-share" tabindex="-1">Python share <a class="header-anchor" href="#python-share" aria-label="Permalink to &quot;Python share&quot;">​</a></h3><p><code>python.shareOfTaskRunTime</code> is the summed task run time of Python stages divided by the summed task run time of all stages; null when no task run time was recorded. It is <code>0</code> for a run with task time and no Python stage. Executor CPU time counts only the JVM task thread, so on a run with a high share it misses the CPU of the Python worker processes; treat <code>executorCpuTimeMs</code> as unreliable there.</p><p>A stage is a Python stage when either signal holds:</p><ul><li>a plan node attributed to it is a Python operator (<code>PythonRDD</code>, <code>BatchEvalPython</code>, <code>ArrowEvalPython</code>, <code>PythonMapInArrow</code>, or a pandas/Arrow grouped or map operator such as <code>FlatMapGroupsInPandas</code>, including suffixed variants such as <code>BatchEvalPythonUDTF</code> and <code>FlatMapGroupsInPandasWithState</code> and the Spark 4.1 <code>ArrowAggregatePython</code> and <code>ArrowWindowPython</code>), which catches Python UDFs and UDTFs inside SQL;</li><li>its name or call site names <code>PythonRDD</code> or <code>org.apache.spark.api.python</code>, which catches RDD lambdas that have no plan and stages that cannot be matched to one.</li></ul><p>The &quot;tasks mostly idle&quot; check that keeps a low-CPU stage from being treated as waiting on an external system uses the same test.</p><h3 id="other-surfaces" tabindex="-1">Other surfaces <a class="header-anchor" href="#other-surfaces" aria-label="Permalink to &quot;Other surfaces&quot;">​</a></h3><p>Every figure the metrics block shares with another surface reads the same core code as that surface. The run comparison (the dashboard&#39;s comparison view, the CLI&#39;s <code>--baseline</code> and the MCP <code>compare_runs</code> tool) uses the same sums for run time, CPU time, GC time, spill, input, output and task count, so a comparison metric equals the matching <code>metrics</code> field for the same run, including the work of failed and speculative task attempts. Per-stage views in the dashboard (the stage table and stage detail) describe the stage&#39;s latest attempt and the task attempts that won, so a stage&#39;s own figures can be smaller than its row in <code>metrics.stages</code>. The dashboard&#39;s Efficiency card reports available capacity (peak concurrent cores times run time), which differs from <code>allocation.coreHours</code> (cores times the hours each executor was alive) under dynamic allocation. The utilization finding&#39;s <code>cpuUtilizationPct</code> is null, as <code>time.executorCpuTimeMs</code> is, when the log recorded no CPU time.</p><h2 id="effective-conf" tabindex="-1">Effective conf <a class="header-anchor" href="#effective-conf" aria-label="Permalink to &quot;Effective conf&quot;">​</a></h2><p>The JSON output also carries <code>effectiveConf</code> (with its own <code>schemaVersion</code>), the Spark properties the run started with, so a script can check that a <code>--conf</code> overlay took effect. It is null when the log records no Spark properties. With <code>--baseline</code> it is inside <code>candidate</code>. The MCP <code>diagnose_run</code> tool returns the same block.</p><ul><li><code>values</code>: property to value, for every property that is not withheld.</li><li><code>maskedKeys</code>: properties present in the log whose value is withheld.</li><li><code>absentKeys</code>: with <code>--conf-keys</code>, the requested properties the log does not contain.</li></ul><p>A value is withheld when its key or value matches Spark&#39;s default secret pattern (<code>(?i)secret|password|token|access[.]?key</code>), as Spark&#39;s own redaction does, when its key names another credential form (<code>passwd</code>, <code>pwd</code>, <code>pass</code>, <code>apiKey</code>, <code>accountKey</code> as in <code>fs.azure.account.key.*</code>, <code>privateKey</code>, <code>sas</code>, <code>sig</code>, <code>credential</code>), when the key or value matches the job&#39;s own <code>spark.redaction.regex</code> (when the log records one; if that pattern uses syntax JavaScript cannot evaluate, every value is withheld), or when the key or value matches <code>--conf-redact-regex &lt;pattern&gt;</code>. A withheld property shows that it is present, not what it is set to, so the output cannot tell you whether a masked property has the value you expected. No hash or other derivative of a withheld value is emitted.</p><p>In every other value, credentials are replaced with <code>[redacted]</code>: in a value that looks like a URL, <code>user:password@host</code> userinfo and the Oracle thin <code>user/password@host</code> form; in any value, <code>name=value</code> parameters named <code>password</code>, <code>passwd</code>, <code>pwd</code>, <code>pass</code>, <code>apikey</code>, <code>accountkey</code>, <code>sas</code>, <code>sig</code>, <code>signature</code> and similar (JDBC and ODBC strings, query strings, JVM options), including signature parameters such as Azure SAS <code>sig=</code> and <code>X-Amz-Signature=</code>. <code>--conf-keys a,b</code> narrows <code>values</code> and <code>maskedKeys</code> to the named properties.</p><p>With <code>--redact</code>, the host names the report pseudonymizes become <code>host-N</code> in every value: the values of <code>*.host</code> and <code>*.hostname</code> properties, executor hosts, and IP or EC2-style addresses. <code>spark.app.name</code> becomes the application&#39;s <code>app-N</code> pseudonym. A host name that appears only in some other property, such as <code>spark.yarn.historyServer.address</code>, is left as is; withhold it with <code>--conf-redact-regex</code> or leave it out with <code>--conf-keys</code>.</p><h2 id="pairing-stages-across-runs" tabindex="-1">Pairing stages across runs <a class="header-anchor" href="#pairing-stages-across-runs" aria-label="Permalink to &quot;Pairing stages across runs {#pairing-stages-across-runs}&quot;">​</a></h2><p>With <code>--baseline</code>, the <code>comparison</code> object pairs the stages of the two runs (<code>stagePairs</code>, with each pair&#39;s <code>deltas</code>) and reports <code>runtimeCoverage</code>, the share of both runs&#39; executor run time that sits in paired stages. <code>confidence</code> is <code>ok</code>, <code>low</code> or <code>insufficient</code>: <code>low</code> when the application names differ or <code>runtimeCoverage</code> is under 0.9, <code>insufficient</code> when neither run recorded any executor run time. <code>confidence</code> does not change an exit code. The fields and the matching rules are in <a href="./../mcp-tools/compare-runs.html"><code>compare_runs</code></a> and <a href="./../run-comparison/how-stages-are-matched.html">How stages are matched</a>.</p><p>When a stage writes to a path that differs per run, for example <code>{sandbox_root}/{session}/{variant}/{batch}</code> where the variant is <code>baseline</code> in one run and <code>c07b</code> in the other, the write stage cannot pair, and it is often the heaviest. Pass <code>--normalize-path &lt;regex&gt;</code>, repeatable, to rewrite what the patterns match, in every plan node&#39;s text, before stages are paired:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">sparkforensics-analyze</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> candidate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --baseline</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> baseline</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --normalize-path</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;/sandbox/[^/]+/[^/]+/&#39;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --format</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> json</span></span></code></pre></div><p>Every match is replaced with a fixed token. Despite the name it is a generic regular expression, not only for paths. It needs <code>--baseline</code> and applies to every candidate of a several-candidates run. It changes which stages pair, and so <code>stagePairs</code>, <code>runtimeCoverage</code> and <code>confidence</code>; findings and the other metrics are untouched. At most 16 patterns of 200 characters each. An invalid pattern, or one that matches the empty string, exits <code>2</code> before any log is read.</p><h2 id="several-budgets-and-several-candidates" tabindex="-1">Several budgets and several candidates <a class="header-anchor" href="#several-budgets-and-several-candidates" aria-label="Permalink to &quot;Several budgets and several candidates&quot;">​</a></h2><h3 id="several-regression-budgets" tabindex="-1">Several regression budgets <a class="header-anchor" href="#several-regression-budgets" aria-label="Permalink to &quot;Several regression budgets&quot;">​</a></h3><p><code>--max-regression-pct</code> checks one metric. To gate on several in one call, repeat <code>--regression-budget &lt;metric&gt;:&lt;pct&gt;</code>, or list them in a file passed with <code>--budgets &lt;file&gt;</code>. Both need <code>--baseline</code>, and each budget takes a <a href="./regression-metric-keys.html#regression-metric-keys">metric key</a> and a percentage of zero or more:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">sparkforensics-analyze</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> candidate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --baseline</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> baseline</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --regression-budget</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> wallClock:10</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --regression-budget</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> gcTime:25</span></span></code></pre></div><p>The <code>--budgets</code> file is JSON with one key, <code>regression</code>, mapping a metric key to its percentage:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;regression&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;wallClock&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">10</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;gcTime&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">25</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;failedTaskRate&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } }</span></span></code></pre></div><p>The CLI refuses to run, with exit code 2 and a message naming the problem, for an unknown top-level key, an unknown metric key, a percentage that is not a non-negative number (in the file a JSON number, not a string; on the flag plain digits such as <code>10</code> or <code>2.5</code>), or a file that can&#39;t be read or isn&#39;t valid JSON.</p><p>The budgets combine like this:</p><ul><li><code>--max-regression-pct</code> with <code>--regression-metric</code> (default <code>wallClock</code>) still works and counts as one more budget next to the new ones.</li><li>Each metric can be budgeted once across the legacy pair, the repeated flag and the file. A metric named twice is a usage error (exit 2), even when both give the same percentage.</li><li>Each budget is checked on its own and gives one <code>max-regression</code> result that carries <code>metric</code>, the key it checks. Any violated budget makes the run exit <code>1</code>; otherwise an inconclusive one makes it exit <code>3</code>. A budget on a metric the baseline or the candidate log can&#39;t provide is inconclusive, never a pass.</li></ul><h3 id="several-candidates" tabindex="-1">Several candidates <a class="header-anchor" href="#several-candidates" aria-label="Permalink to &quot;Several candidates&quot;">​</a></h3><p>Pass two or more logs as positional arguments, with <code>--baseline</code>, to compare each against the same baseline. The baseline is parsed once. Output is NDJSON: one line per candidate, in argument order, written as each one finishes. <code>--out &lt;path&gt;</code> writes the lines to a file instead of stdout, and <code>--format ndjson</code> selects this output for a single candidate too. The mode can&#39;t be combined with <code>--export-html</code>, <code>--shs-base-url</code>, <code>--format json</code> or <code>--format md</code> (exit 2). <code>--redact</code>, <code>--thresholds</code>, <code>--impact</code>, <code>--type</code>, <code>--stage</code>, <code>--conf-keys</code>, <code>--conf-redact-regex</code> and every budget flag apply to each candidate.</p><p>Each line is one JSON object:</p><table tabindex="0"><thead><tr><th>Field</th><th>Meaning</th></tr></thead><tbody><tr><td><code>log</code></td><td>The candidate path, as given on the command line. With <code>--redact</code>, <code>candidate-&lt;n&gt;</code> instead, <code>n</code> being the candidate&#39;s 1-based position.</td></tr><tr><td><code>status</code></td><td><code>pass</code>, <code>violation</code>, <code>inconclusive</code> or <code>error</code>.</td></tr><tr><td><code>exitCode</code></td><td>The exit code this line alone would give: <code>0</code> for <code>pass</code>, <code>1</code> for <code>violation</code>, <code>3</code> for <code>inconclusive</code>, and for <code>error</code> <code>4</code> (the log can&#39;t be read or parsed) or <code>6</code> (an internal failure while analyzing it).</td></tr><tr><td><code>error</code></td><td>The message when <code>status</code> is <code>error</code>; otherwise <code>null</code>. With <code>--redact</code>, a generic message that names no path.</td></tr><tr><td><code>budgets</code></td><td>This candidate&#39;s budget results, each with <code>name</code>, <code>status</code> (<code>pass</code>, <code>violation</code> or <code>inconclusive</code>) and <code>detail</code>, plus <code>metric</code> on <code>max-regression</code>. Empty for an <code>error</code> line.</td></tr><tr><td><code>candidate</code></td><td>The candidate&#39;s report with its <code>metrics</code> and <code>effectiveConf</code> blocks, the same object the single-candidate JSON output carries under <code>candidate</code>. <code>null</code> for an <code>error</code> line.</td></tr><tr><td><code>comparison</code></td><td><code>verdict</code>, <code>confidence</code>, <code>reason</code>, <code>matchedCoverage</code>, <code>runtimeCoverage</code>, <code>metrics</code>, <code>findings</code>, <code>comparisonSchemaVersion</code>, <code>stagePairs</code>, <code>unmatched</code>, <code>replanned</code> and <code>bookkeepingStageIds</code>, the same object the single-candidate JSON output carries under <code>comparison</code>. <code>null</code> for an <code>error</code> line.</td></tr></tbody></table><p>A line&#39;s <code>status</code> follows the single-candidate rules: <code>violation</code> if any budget result is a violation, else <code>inconclusive</code> if any is inconclusive (including the <code>run-complete</code> check for a log with no <code>ApplicationEnd</code>), else <code>pass</code>.</p><p>A candidate that can&#39;t be read or parsed doesn&#39;t stop the others. Its line has <code>status: &quot;error&quot;</code>, <code>exitCode: 4</code>, the message in <code>error</code>, and <code>null</code> for <code>candidate</code> and <code>comparison</code>, as a single-candidate run exits <code>4</code> for a candidate it can&#39;t parse. The process exit code is the worst line, in the order <code>6</code>, <code>5</code>, <code>4</code>, <code>1</code>, <code>3</code>, <code>0</code>, so an unreadable candidate outranks a violation and a violation outranks an inconclusive result.</p><p>Two failures stop the batch before any line is written:</p><ul><li>A usage error (bad flags or arguments, an invalid <code>--thresholds</code> or <code>--budgets</code> file) exits <code>2</code>.</li><li>A baseline that can&#39;t be read or parsed exits <code>5</code>. Fix the baseline and run the batch again.</li></ul><p>An internal failure that isn&#39;t tied to one candidate, such as an unwritable <code>--out</code> path, exits <code>6</code>.</p><p>With <code>--redact</code>, no candidate path is written anywhere, since event-log file names usually carry the app id. <code>log</code> and the <code>stderr</code> line prefixes name each candidate by its 1-based position (<code>candidate-1</code>, <code>candidate-2</code>, ...), and an <code>error</code> line carries a generic message instead of the parser&#39;s, which may quote the path: <code>Candidate 2 could not be read or parsed.</code>, or <code>could not be analyzed</code> for an internal failure. An unreadable baseline prints <code>The baseline could not be read or parsed.</code> to <code>stderr</code>. A single-candidate run does the same, naming <code>The baseline</code> or <code>The candidate</code>.</p>`,82)])])}const g=t(d,[["render",i]]);export{u as __pageData,g as default};

package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.DpnfKz3x.lean.js

@@ -0,0 +1 @@
+import{_ as t,o as a,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"CI and automation","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/ci-and-automation.md","filePath":"user-guide/getting-started/ci-and-automation.md"}'),d={name:"user-guide/getting-started/ci-and-automation.md"};function i(n,e,c,r,l,h){return a(),o("div",null,[...e[0]||(e[0]=[s("",82)])])}const g=t(d,[["render",i]]);export{u as __pageData,g as default};

package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as n}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Reading the dashboard","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/reading-the-dashboard.md","filePath":"user-guide/getting-started/reading-the-dashboard.md"}'),s={name:"user-guide/getting-started/reading-the-dashboard.md"};function r(i,e,d,h,c,l){return o(),a("div",null,[...e[0]||(e[0]=[n('<h1 id="reading-the-dashboard" tabindex="-1">Reading the dashboard <a class="header-anchor" href="#reading-the-dashboard" aria-label="Permalink to &quot;Reading the dashboard&quot;">​</a></h1><p>How to read the dashboard once a run is loaded, including the advanced view and the rest of the top bar.</p><p>When parsing finishes, the dashboard shows a board of widgets, each in its own card with a title. Every widget that flags a problem uses the same convention: a colored impact dot (critical / warning / info) and an ALL-CAPS tag for the bottleneck category (<code>SKEW</code>, <code>SPILL</code>, <code>GC</code>, and so on: see <a href="./../understanding-findings.html">Understanding findings</a>). Widgets list every affected stage, not just the worst one. For a finding with a run-time estimate, the dot&#39;s color tracks how much run time it could save rather than how unusual the metric looks, so a small-looking anomaly with a big payoff can outrank a dramatic one that would barely move your run time. A finding with only a resource estimate, or none, keeps the level its check assigned.</p><p>The board opens with a verdict: one line saying where to start, a short summary of what was found, and up to three numbered next steps. Each step says what the finding measured and what to try, and has a <strong>Show evidence</strong> button that jumps to the finding&#39;s detail widget. Findings on the same stage are folded into one step, because they usually share a cause and their savings overlap rather than add up. Each savings figure says what it counts: a time such as &quot;58.6s of run time&quot; is how much sooner the run could finish, while a resource figure such as &quot;3.0 GB-h of unused executor memory&quot; or &quot;0.7 core-h of core time&quot; is cluster time a fix would free up, which cuts cost but may not shorten the run. A figure &quot;of idle core capacity&quot; counts allocated cores that ran no task, not work a fix removes. <strong>Copy next steps</strong> copies the whole plan as a plain checklist (run, verdict, and numbered steps with their stage and savings) to paste into a ticket or a message. Steps follow the same savings ranking as the rest of the board. When much of the run&#39;s executor capacity sat idle, the summary says so, without moving cluster size ahead of a bigger fix.</p><p>The verdict also says how the run ended. When every job succeeded it says so. When a job failed, the title says the run failed (or how many of its jobs did), quotes the first line of the reason Spark recorded, and puts the failure first in the next steps, ahead of any speed-up, since a job has to finish before its speed matters.</p><p>A run is called clean only when the log had everything its checks need. When something was missing, the run is never called clean: the Findings tab&#39;s <strong>Clean checks</strong> list says which checks could not run and, where Spark has one, the setting to turn on for the next run (for example <code>spark.eventLog.logStageExecutorMetrics=true</code> for per-executor memory). If nothing else was found, the verdict title says so too.</p><p>A run scorecard sits under the verdict: <strong>Wall-clock</strong> (total run time), <strong>Efficiency</strong> (the share of that time with a stage running; higher is better; <strong>Not measured</strong> when no stage in the log recorded an end) and <strong>Unused core time</strong> (the share of executor core time that ran no task, the same idle figure a verdict step reports; lower is better). When the log has no complete application timing, one <strong>Timing unavailable</strong> notice replaces the three tiles. A collapsed <strong>New to Spark tuning?</strong> primer in the verdict explains stages, tasks, executors, shuffle and how to read savings. Stage labels such as <strong>Stage 7</strong> open that stage&#39;s details: how long it ran and what share of the run that was, then each of its findings with what it measured, what to try and a <strong>Show evidence</strong> button, followed by its overview, task, locality, I/O, spill, GC and plan sections (spill and GC only when the stage had them). Below it, two tabs split the rest of the board:</p><ol><li><strong>Findings</strong>: every flagged finding and its detail widget, grouped by impact band (Critical, Warning, Info). Within a band, a recommendation row for a bottleneck type collapses into a summary row when it fires more than once; clicking the summary row expands its full list, and clicking any single row or widget jumps straight to that finding. Each band leads with its recommendation rows; its detail widgets (the charts and per-stage numbers behind them) sit under <strong>Show the evidence</strong>, and <strong>Show evidence</strong> on any finding opens them for you. Advanced view shows the widgets without that step. Widgets that found nothing fold away into a &quot;Clean checks&quot; disclosure. A check the log lacked the data for is listed there under <strong>Not checked on this log</strong>, not as a pass, with the reason and the setting to turn on: for example every per-stage check when no stage finished, or core usage, memory and executor churn when the log has no end-of-run record. This is the tab you land on.</li><li><strong>Full app report</strong>: the wall-clock and executor timelines, the stage table, and the reference-only cards, led by core usage by locality, which shows even on a clean run.</li></ol><p>Click a finding&#39;s tag to open its reference material in a slide-in panel beside the dashboard: the dashboard stays visible and interactive, so you can check a metric against the reference without losing your place. Esc closes the panel, even while you are reading or scrolling inside it. The topbar&#39;s <strong>Docs</strong> button opens these docs in a new tab.</p><h2 id="advanced-view" tabindex="-1">Advanced view <a class="header-anchor" href="#advanced-view" aria-label="Permalink to &quot;Advanced view&quot;">​</a></h2><p>The topbar has an <strong>Advanced view</strong> toggle. It&#39;s off by default, which keeps each widget to the finding itself and what to do about it. Turn it on to also show confidence levels, supporting evidence, and a page icon beside each finding&#39;s tag that opens its entry in these docs, plus a few extra table columns and the finding filter bar (impact, type, stage). A filter that is already active, for example from a shared link, keeps the filter bar visible either way. In the verdict, each step also says how its savings figure was estimated (measured or modeled, and whether the stage ran alone or shared the cluster, which makes the figure a range from a floor to an optimistic high), shows any confidence marker, and a line states how the steps are ordered. The scorecard switches from plain captions to the raw run and idle-time breakdown, and the newcomer primer is hidden. Advanced view also turns on single-key triage shortcuts: <code>j</code> and <code>k</code> step through the verdict&#39;s steps and then every finding row, <code>Enter</code> shows the focused finding&#39;s evidence, or expands a grouped finding, <code>f</code> jumps to the filters, and <code>1</code> and <code>2</code> switch between <strong>Findings</strong> and <strong>Full app report</strong>. They stay off in the default view, so they never surprise a first-time visitor or a screen-reader user. Your choice is remembered across runs.</p><h2 id="the-rest-of-the-topbar" tabindex="-1">The rest of the topbar <a class="header-anchor" href="#the-rest-of-the-topbar" aria-label="Permalink to &quot;The rest of the topbar&quot;">​</a></h2><p>Once a run is loaded, the topbar also carries a few more controls. The run name opens a menu of recent files, to switch to another run or load a new file. When the parser skipped malformed lines, a warning beside the name says how many. The count chip (&quot;4 critical&quot;) counts the same findings the verdict ranks; click it to jump to that band of the Findings list (a board filter hiding the band is cleared, with a notice saying so). It reads <strong>Run failed</strong> when a job failed and there is no finding to count, <strong>No findings</strong> only when the verdict calls the run clean, and <strong>Not fully checked</strong> otherwise. For keyboard users, the first Tab stop is <strong>Skip to the verdict</strong>. <strong>New analysis</strong> goes back to the landing page to load another run; dropping a file onto the dashboard loads it too. <strong>Compare with another run</strong> keeps this run as the baseline and asks only for the other one (see <a href="./../run-comparison.html">Run comparison mode</a>). <strong>Plan graph</strong> opens an interactive node-and-edge view of the run&#39;s SQL execution plan, filterable down to I/O operators (scan, exchange), a broader &quot;basic&quot; set, or every operator. <strong>Export evidence</strong> downloads the current run&#39;s findings as a portable Markdown or JSON report, the same report the CLI produces, or as <strong>Download HTML dashboard</strong>: one <code>.html</code> file holding this dashboard for the run, which opens in any browser with no server, like the CLI&#39;s <code>--export-html</code> folder. Exported dashboards carry no docs links: finding tags and &quot;learn more&quot; references show as plain text, so look them up in these docs yourself. An exported dashboard shows the conclusions worked out when it was exported, and its footer names the tool, core version and build that produced it; it has no <strong>Export evidence</strong> menu of its own. A file written by an incompatible release opens to a message saying so; export the run again with your current release. Turn on <strong>Redact identifiers</strong> first if the export is headed outside the environment that produced it, since that pseudonymizes the app id, the app name and any host/IP tokens in all three formats. <strong>Keyboard shortcuts</strong> (press <code>?</code> from anywhere) lists every shortcut. Rounding it out: a <strong>Docs</strong> link and a theme toggle.</p>',13)])])}const p=t(s,[["render",r]]);export{u as __pageData,p as default};

package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js

@@ -0,0 +1 @@
+import{_ as t,o,c as a,a5 as n}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Reading the dashboard","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/reading-the-dashboard.md","filePath":"user-guide/getting-started/reading-the-dashboard.md"}'),s={name:"user-guide/getting-started/reading-the-dashboard.md"};function r(i,e,d,h,c,l){return o(),a("div",null,[...e[0]||(e[0]=[n("",13)])])}const p=t(s,[["render",r]]);export{u as __pageData,p as default};

package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.js

@@ -0,0 +1 @@
+import{_ as o,o as t,c as s,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Regression metric keys","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/regression-metric-keys.md","filePath":"user-guide/getting-started/regression-metric-keys.md"}'),i={name:"user-guide/getting-started/regression-metric-keys.md"};function c(d,e,a,l,n,u){return t(),s("div",null,[...e[0]||(e[0]=[r('<h1 id="regression-metric-keys" tabindex="-1">Regression metric keys <a class="header-anchor" href="#regression-metric-keys" aria-label="Permalink to &quot;Regression metric keys&quot;">​</a></h1><p>The metric keys that a regression metric or regression budget accepts.</p><p><code>--regression-metric</code>, <code>--regression-budget</code> and the <code>--budgets</code> file (and the MCP <code>evaluate_budgets</code> tool&#39;s <code>regressionMetric</code>) take one of these keys; the default for <code>--regression-metric</code> is <code>wallClock</code>:</p><ul><li><code>wallClock</code>: wall-clock duration</li><li><code>executorRunTime</code>: summed executor run time</li><li><code>shuffleSpill</code>: memory spill (Spark&#39;s <code>memoryBytesSpilled</code>)</li><li><code>diskSpill</code>: disk spill (Spark&#39;s <code>diskBytesSpilled</code>)</li><li><code>gcTime</code>: JVM GC time</li><li><code>taskSkew</code>: p95 task skew</li><li><code>failedTaskRate</code>: failed-task rate</li><li><code>executorCpuTime</code>: summed executor CPU time, in milliseconds (unavailable for a run whose log never recorded it)</li><li><code>allocatedCoreHours</code>: executor cores times hours alive, see <a href="./ci-and-automation.html#allocation">Allocation</a></li></ul><p>Four more keys, <code>inputBytes</code>, <code>outputBytes</code>, <code>taskCount</code> and <code>executorsAdded</code>, measure workload volume rather than performance. They have no better or worse direction, so a regression budget on one of them reports <code>inconclusive</code> whenever the value changes, and passes when it doesn&#39;t.</p>',5)])])}const p=o(i,[["render",c]]);export{g as __pageData,p as default};

package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.lean.js

@@ -0,0 +1 @@
+import{_ as o,o as t,c as s,a5 as r}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Regression metric keys","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/regression-metric-keys.md","filePath":"user-guide/getting-started/regression-metric-keys.md"}'),i={name:"user-guide/getting-started/regression-metric-keys.md"};function c(d,e,a,l,n,u){return t(),s("div",null,[...e[0]||(e[0]=[r("",5)])])}const p=o(i,[["render",c]]);export{g as __pageData,p as default};

package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js

@@ -0,0 +1,4 @@
+import{_ as s,o as t,c as a,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Tuning detector thresholds","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/tuning-thresholds.md","filePath":"user-guide/getting-started/tuning-thresholds.md"}'),i={name:"user-guide/getting-started/tuning-thresholds.md"};function d(n,e,r,h,l,c){return t(),a("div",null,[...e[0]||(e[0]=[o(`<h1 id="tuning-detector-thresholds" tabindex="-1">Tuning detector thresholds <a class="header-anchor" href="#tuning-detector-thresholds" aria-label="Permalink to &quot;Tuning detector thresholds&quot;">​</a></h1><p>How to run a detector check with your own threshold instead of its default.</p><p>Every check fires at a fixed default threshold. When your normal workload trips one on purpose, such as a join you skew deliberately, pass <code>--thresholds &lt;file&gt;</code> to run that check with your own value. The file is JSON, keyed by detector, then by threshold name:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;skew&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;ratioWarn&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">6</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;shuffle&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;minBytes&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1073741824</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>The names, units and defaults are the <code>detectors</code> catalog in the CLI&#39;s JSON report (<code>thresholds</code> on each row): bytes are raw byte counts, times are milliseconds, and <code>*Pct</code>/<code>*Rate</code> values are fractions (<code>0.05</code> is 5%). A tier list such as <code>slowHost.ratioTiers</code> takes the same number of values, in ascending order. The <code>configAudit</code> checks can&#39;t be tuned: they compare your Spark settings against Spark&#39;s own defaults.</p><p>The CLI refuses to run, with exit code 2 and a message naming the problem, when the file can&#39;t be read, isn&#39;t valid JSON, or names an unknown detector or threshold, or a value of the wrong shape. It never falls back to the defaults silently.</p><p>A tuned run says so wherever it reports:</p><ul><li>Each finding from a tuned detector carries <code>tunedThresholds</code> (each overridden threshold&#39;s <code>value</code> and <code>default</code>), and its <code>validationRequired</code> text names them and, when the finding has an impact estimate, says that estimate is unvalidated. The estimates are calibrated against the default thresholds, so they were never checked for a finding your override lets through. Tuning <code>slowHost</code> also labels <code>stageSlowness</code> findings (as <code>slowHost.&lt;name&gt;</code>), because a slow host hides a stage&#39;s slowness finding, so the override decides which of those you see.</li><li>A clean check measured against a tuned threshold carries <code>tunedThresholds</code> too, with the tuned value and the default. Its <code>thresholdSummary</code> shows the tuned numbers for most detectors (for example <code>skew</code>, <code>gc</code>, <code>spill</code>, <code>slowHost</code>). A few summaries are fixed text with no number to tune.</li><li><code>summary.tunedThresholds</code> lists every tuned detector, and each tuned row of the <code>detectors</code> catalog shows the thresholds the run used.</li><li>The Markdown report adds a <code>Tuned thresholds</code> line to its header and a <code>tuned thresholds</code> line to each affected finding.</li></ul><p>An override equal to the default changes nothing and is not labeled. <code>--baseline</code> runs the baseline with the same overrides, so the comparison compares like with like. The budget flags are separate gates computed from the run&#39;s own figures; the one that recomputes a detector&#39;s figure, <code>--max-skew</code>, uses the file&#39;s <code>skew.minTasksForP95</code>, so it measures the same ratio the skew finding reports. <code>--export-html</code> writes the dashboard with the default thresholds, because the dashboard never tunes, and prints a note to stderr saying so. The browser dashboard has no tuning.</p>`,9)])])}const g=s(i,[["render",d]]);export{p as __pageData,g as default};

package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js

@@ -0,0 +1 @@
+import{_ as s,o as t,c as a,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Tuning detector thresholds","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started/tuning-thresholds.md","filePath":"user-guide/getting-started/tuning-thresholds.md"}'),i={name:"user-guide/getting-started/tuning-thresholds.md"};function d(n,e,r,h,l,c){return t(),a("div",null,[...e[0]||(e[0]=[o("",9)])])}const g=s(i,[["render",d]]);export{p as __pageData,g as default};

package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.js

@@ -0,0 +1,16 @@
+import{_ as s,o as t,c as a,a5 as i}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"MCP tools reference","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools.md","filePath":"user-guide/mcp-tools.md"}'),o={name:"user-guide/mcp-tools.md"};function n(d,e,r,l,c,h){return t(),a("div",null,[...e[0]||(e[0]=[i(`<h1 id="mcp-tools-reference" tabindex="-1">MCP tools reference <a class="header-anchor" href="#mcp-tools-reference" aria-label="Permalink to &quot;MCP tools reference&quot;">​</a></h1><p>SparkForensics has an MCP server with eight tools, so an MCP-aware client (or an AI agent) can diagnose a run without opening the dashboard.</p><p>Five of the eight tools (<code>list_runs</code>, <code>diagnose_run</code>, <code>get_run_summary</code>, <code>compare_runs</code>, <code>get_finding_evidence</code>) accept an optional <code>redact: boolean</code> parameter (default <code>false</code>) that pseudonymizes the app id and any host/IP tokens in the response (<code>app-1</code>, <code>host-1</code>, ...), so a result can be shared outside the environment that produced it. The app name is as identifying as the id, so every tool replaces it with the app id&#39;s pseudonym. It also pseudonymizes the value of any field named <code>host</code>, and drops the message and stack text of failed-task errors and the failure reason of a failed stage or job, which can carry file paths and data values that no pattern recognizes. On <code>compare_runs</code>, <code>runIdA</code>/<code>runIdB</code> are caller-supplied identifiers, not Spark application ids, so there&#39;s no single app-id field to redact; <code>redact</code> instead scans stage names and other free text for embedded app ids and host/IP tokens and pseudonymizes those. <code>list_runs</code> pseudonymizes every run&#39;s app id and name consistently, so two attempts of the same app still redact to the same identity across the list.</p><h2 id="connecting-a-client" tabindex="-1">Connecting a client <a class="header-anchor" href="#connecting-a-client" aria-label="Permalink to &quot;Connecting a client&quot;">​</a></h2><p>Run the server with <code>npx</code>:</p><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>npx sparkforensics-mcp</span></span></code></pre></div><p>Or point an MCP client (Claude Desktop, Claude Code) at it with this config:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;mcpServers&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;sparkforensics&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;command&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;npx&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;args&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;sparkforensics-mcp&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>To tune detector thresholds, start the server with <code>--thresholds &lt;file&gt;</code> (add <code>&quot;--thresholds&quot;, &quot;/absolute/path/thresholds.json&quot;</code> to <code>args</code>). The file format is the CLI&#39;s: see <a href="./getting-started/tuning-thresholds.html#tuning-detector-thresholds">Tuning detector thresholds</a>. The overrides apply to every tool for the life of the server; a client can&#39;t change them per call. An unreadable or invalid file stops the server from starting: it exits with status 2, with the problem on stderr. <code>sparkforensics-mcp --help</code> prints the usage and the tool list. On a tuned server, <code>diagnose_run</code>, <code>compare_runs</code> and <code>evaluate_budgets</code> add a top-level <code>tunedThresholds</code>, and each finding and clean check from a tuned detector carries its own <code>tunedThresholds</code>, as in the CLI report. The Markdown output of <code>diagnose_run</code> and <code>compare_runs</code> names the tuned thresholds too.</p><h3 id="connecting-over-http" tabindex="-1">Connecting over HTTP <a class="header-anchor" href="#connecting-over-http" aria-label="Permalink to &quot;Connecting over HTTP&quot;">​</a></h3><p>The local server (<code>npx sparkforensics-server</code>, see <a href="./getting-started.html#local-server-mode">Run it locally</a>) also serves the same eight tools over MCP&#39;s streamable HTTP transport at <code>http://127.0.0.1:4173/mcp</code> (or the port you pass with <code>--port</code>). Point a client that speaks streamable HTTP, such as Claude Code, at that URL:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;mcpServers&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;sparkforensics&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;type&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;http&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;url&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;http://127.0.0.1:4173/mcp&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>It is for clients on the same machine only. The server listens on 127.0.0.1, and a request whose <code>Host</code> header isn&#39;t <code>127.0.0.1:&lt;port&gt;</code> or <code>localhost:&lt;port&gt;</code> gets a 403, which blocks DNS-rebinding attacks from web pages. Use <code>127.0.0.1</code> in the URL: <code>localhost</code> can resolve to the IPv6 address <code>::1</code>, where the server doesn&#39;t listen.</p><p>The endpoint always runs the default detector thresholds: the server takes no <code>--thresholds</code> flag. To tune thresholds, run <code>sparkforensics-mcp</code> over stdio instead. Runs are cached in the server process as they are for <code>sparkforensics-mcp</code>, so a <code>runId</code> from one request works in later ones until it expires. A relative <code>source.path</code> resolves against the directory you started the server from. The examples below use short relative names; pass an absolute path when in doubt.</p><h2 id="errors" tabindex="-1">Errors <a class="header-anchor" href="#errors" aria-label="Permalink to &quot;Errors&quot;">​</a></h2><p>Errors a tool raises carry <code>structuredContent.code</code>, the same way in all eight tools:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;isError&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;content&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;type&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;text&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;text&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;&lt;error message&gt;&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;structuredContent&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;code&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;&lt;error-code&gt;&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Arguments that fail the input schema (a wrong type, a missing required field, both <code>dir</code> and <code>shsBaseUrl</code>, <code>maxResults</code> below 1) return <code>isError: true</code> with the text <code>MCP error -32602: Input validation error: ...</code> and no <code>structuredContent</code>.</p><p>The codes:</p><ul><li><code>run-not-found</code>: no cached run for the <code>runId</code> you passed.</li><li><code>finding-not-found</code>: that <code>findingId</code> isn&#39;t on that run.</li><li><code>invalid-date-filter</code>: <code>list_runs</code>&#39;s <code>minDate</code> or <code>maxDate</code> isn&#39;t a parseable date.</li><li><code>invalid-type</code>: that finding <code>type</code> isn&#39;t one <code>get_finding_documentation</code> recognizes.</li><li><code>invalid-anchor</code>: that <code>anchor</code> doesn&#39;t resolve to a known <code>get_reference_doc</code> page.</li><li><code>invalid-event-log</code>: the file doesn&#39;t exist, the file isn&#39;t a decodable event log, the folder isn&#39;t a rolling event-log directory, or the History Server archive couldn&#39;t be decoded.</li><li><code>application-not-found</code>: the History Server returned a 404 for that <code>appId</code>/<code>attemptId</code>.</li><li><code>upstream-unreachable</code>: loading a run from the History Server failed because the server couldn&#39;t be reached (connection refused, DNS failure), didn&#39;t send headers in time, or stalled mid-body. <code>list_runs</code> reports it when the server can&#39;t be reached or doesn&#39;t answer in time. <code>SPARKFORENSICS_SHS_TIMEOUT_MS</code> (default 30000) sets these timeouts. If the server is unreachable entirely (an SSH-only cluster), see <a href="./alternative-log-retrieval/ssh-bastion.html#behind-an-ssh-bastion">Behind an SSH bastion</a>.</li><li><code>archive-too-large</code>: the History Server archive blew the byte cap (1 GiB by default). Override it with <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code>.</li><li><code>directory-not-found</code>: <code>list_runs</code>&#39;s <code>dir</code> doesn&#39;t exist, isn&#39;t a directory, or isn&#39;t readable.</li><li><code>invalid-shs-base-url</code>: <code>list_runs</code>&#39;s <code>shsBaseUrl</code> isn&#39;t an absolute HTTP(S) URL without credentials, query, or fragment.</li><li><code>access-or-upstream-failure</code>: the fallback code. Bad parameters, a failed History Server fetch, or any error that carries no more specific code. Common causes: neither <code>source</code> nor <code>runId</code> given, neither <code>dir</code> nor <code>shsBaseUrl</code> given, <code>regressionMetric</code> without <code>maxRegressionPct</code>, and a <code>source</code> whose <code>shsBaseUrl</code> or <code>appId</code> isn&#39;t valid.</li></ul>`,20)])])}const k=s(o,[["render",n]]);export{u as __pageData,k as default};

package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.lean.js

@@ -0,0 +1 @@
+import{_ as s,o as t,c as a,a5 as i}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"MCP tools reference","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools.md","filePath":"user-guide/mcp-tools.md"}'),o={name:"user-guide/mcp-tools.md"};function n(d,e,r,l,c,h){return t(),a("div",null,[...e[0]||(e[0]=[i("",20)])])}const k=s(o,[["render",n]]);export{u as __pageData,k as default};

package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.CPowlb2q.js

@@ -0,0 +1,55 @@
+import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"compare_runs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/compare-runs.md","filePath":"user-guide/mcp-tools/compare-runs.md"}'),n={name:"user-guide/mcp-tools/compare-runs.md"};function h(l,s,o,d,p,k){return a(),e("div",null,[...s[0]||(s[0]=[t(`<h1 id="compare-runs" tabindex="-1"><code>compare_runs</code> <a class="header-anchor" href="#compare-runs" aria-label="Permalink to &quot;\`compare_runs\`&quot;">​</a></h1><p>Reference for the <code>compare_runs</code> tool of the SparkForensics MCP server.</p><p>Compare two runs: the comparison page&#39;s verdict, metric deltas (<code>metrics</code>) and categorized findings delta (<code>findings</code>), and the share of executor run time in paired stages (<code>runtimeCoverage</code>). The result carries the same <code>verdict</code>, <code>confidence</code>, <code>reason</code>, <code>matchedCoverage</code>, <code>runtimeCoverage</code>, <code>metrics</code>, <code>findings</code>, <code>comparisonSchemaVersion</code>, <code>unmatched</code>, <code>replanned</code> and <code>bookkeepingStageIds</code> as the <code>comparison</code> object of the CLI&#39;s <code>--baseline</code> JSON output, plus <code>stagePairs</code> when <code>include</code> asks for it (the CLI always carries it), plus <code>runIdA</code> and <code>runIdB</code>. <code>metricDeltas</code> and <code>findingsDelta</code> repeat <code>metrics</code> and <code>findings</code> under their earlier names; they are deprecated and will be removed in a later release. <code>verdict</code> is the headline the dashboard&#39;s comparison page opens with (the baseline is <code>runIdA</code>/<code>sourceA</code>, the candidate <code>runIdB</code>/<code>sourceB</code>): a <code>title</code> such as &quot;The candidate finished 9.9s faster than the baseline (37%)&quot;, or a failed-job headline when either run had jobs fail, a <code>tone</code> (<code>better</code>, <code>worse</code>, <code>same</code> or <code>unknown</code>), and <code>sentences</code> naming which cost metrics and finding categories moved each way. When either log has no <code>ApplicationEnd</code> event, the title compares how much run time each log covers (&quot;The candidate&#39;s log covers 2.0s less run time than the baseline&#39;s&quot;) and the tone is <code>unknown</code>.</p><p>Parameters:</p><ul><li><code>runIdA</code> / <code>sourceA</code>: identify the baseline (same <code>source</code> shape as above)</li><li><code>runIdB</code> / <code>sourceB</code>: identify the candidate</li><li><code>redact</code>: <code>boolean</code> (default <code>false</code>), pseudonymizes any app id or host/IP tokens embedded in free text (stage names and similar) in the response: see the note at the top of this page</li><li><code>include</code>: <code>[&quot;stagePairs&quot;]</code> (optional), adds <code>stagePairs</code>, one row per paired stage. It is left out by default because it grows with the number of stages.</li><li><code>normalizePath</code>: <code>string[]</code> (optional), regular expressions whose every match in a plan node&#39;s text is replaced with a fixed token before stages are paired, so text that is specific to your runs does not keep the same stage from pairing. Despite the name it is a generic pattern, not only for paths. It affects stage pairing, <code>runtimeCoverage</code> and <code>confidence</code>, never the findings. At most 16 patterns of 200 characters each. An invalid pattern, or one that matches the empty string, fails the call before any log is read. A pattern that backtracks catastrophically can stall the server, and the length cap does not prevent that: an agent passing patterns should keep them simple and anchored.</li><li><code>format</code>: <code>&quot;json&quot; | &quot;md&quot;</code> (default <code>&quot;json&quot;</code>), switches <code>content[0].text</code> to a rendered Markdown report instead of JSON. <code>structuredContent</code> always stays JSON-shaped, regardless of <code>format</code>.</li></ul><p>Each side needs either a <code>runId</code> or a <code>source</code>, and you can mix them: a cached run ID for the baseline, a fresh file for the candidate.</p><p>Example call (add <code>&quot;include&quot;: [&quot;stagePairs&quot;]</code> to the arguments for the pair rows):</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;compare_runs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;sourceA&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;baseline.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;sourceB&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;candidate.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Example response with <code>include: [&quot;stagePairs&quot;]</code> (<code>metrics</code> and <code>metricDeltas</code> trimmed to their first row of 13, and each pair&#39;s <code>deltas</code> to three of its eight metrics):</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;runIdA&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;aaaa1111-...&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;runIdB&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;bbbb2222-...&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;verdict&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;title&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;The candidate finished 1.0s faster than the baseline (50%)&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;tone&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;better&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;sentences&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;confidence&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;ok&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;reason&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;matchedCoverage&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;runtimeCoverage&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;metrics&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;key&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;wallClock&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;label&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Wall-clock duration&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;baseline&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">2000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;candidate&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;delta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">-1000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;direction&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;improvement&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;findings&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;introduced&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;resolved&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;comparisonSchemaVersion&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;stagePairs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;pairId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;b0-c0&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;baseStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;candStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;quality&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;exact&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;score&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;deltas&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">        &quot;executorRunTime&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseline&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1800</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candidate&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">900</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;delta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">-900</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">        &quot;executorCpuTime&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseline&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candidate&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;delta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">        &quot;memoryBytesSpilled&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseline&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candidate&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;delta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;unmatched&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;replanned&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;bookkeepingStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;baseStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;candStageIds&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;metricDeltas&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;key&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;wallClock&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;label&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Wall-clock duration&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;baseline&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">2000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;candidate&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;delta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">-1000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;direction&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;improvement&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;findingsDelta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;introduced&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;resolved&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each <code>findings.introduced</code>/<code>resolved</code> row is <code>{ rule, type, impactBand, baseCount, candCount, delta, stages }</code>, with <code>stages</code> naming the affected stages. A <code>metrics</code> row&#39;s <code>direction</code> is <code>improvement</code>, <code>regression</code>, <code>unchanged</code>, <code>neutral</code> (a volume metric, where more isn&#39;t worse) or <code>unavailable</code>, and an unavailable row carries an <code>unavailableReason</code>.</p><p>The comparison block (<code>comparisonSchemaVersion</code> <code>1</code>) holds:</p><ul><li><code>stagePairs</code> (only with <code>include</code>): one entry per pair of stages, <code>pairId</code> (stable for one pair of runs; key on it), <code>baseStageIds</code> and <code>candStageIds</code>, <code>quality</code> (<code>exact</code>, <code>structural</code> or <code>aligned</code>), <code>score</code> (0 to 1) and <code>deltas</code>. <code>deltas</code> has one <code>{ baseline, candidate, delta }</code> entry for each of <code>executorRunTime</code>, <code>executorCpuTime</code> (ms), <code>memoryBytesSpilled</code>, <code>diskBytesSpilled</code>, <code>inputBytes</code>, <code>outputBytes</code>, <code>shuffleReadBytes</code> and <code>shuffleWriteBytes</code>. The figures count every task attempt of the stage, failed ones included, the same sums the whole-run <code>metrics</code> use; a figure the log did not record is <code>null</code>. A stage&#39;s own record in the run report keeps the latest attempt&#39;s figures.</li><li><code>unmatched</code>: <code>{ baseStageIds, candStageIds }</code>, the stages that paired with nothing.</li><li><code>replanned</code>: groups of stages whose plan changed between the runs. Always an empty array.</li><li><code>bookkeepingStageIds</code>: <code>{ baseStageIds, candStageIds }</code>, stages that only read the Delta log or its checkpoints. They are in no pair, not in <code>unmatched</code> and not in the coverage.</li><li><code>runtimeCoverage</code>: the share of both runs&#39; executor run time (bookkeeping stages excluded) in paired stages, or <code>null</code> when neither run recorded any.</li><li><code>matchedCoverage</code>: the share of stages paired by exact match, by count.</li></ul><p>Every pair has the quality <code>exact</code>; <code>structural</code> and <code>aligned</code> are reserved values. See <a href="./../run-comparison/how-stages-are-matched.html">How stages are matched</a>.</p><p><code>confidence</code> is <code>ok</code>, <code>low</code> or <code>insufficient</code>, and <code>reason</code> says why when it is not <code>ok</code>. It is <code>low</code> when the two runs&#39; names differ, or when under 90% of executor run time is in paired stages (<code>runtimeCoverage</code>), for example &quot;Only 62% of executor run time is in matched stages&quot;. It is <code>insufficient</code> when neither run recorded any executor run time.</p>`,15)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};

package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.CPowlb2q.lean.js

@@ -0,0 +1 @@
+import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"compare_runs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/mcp-tools/compare-runs.md","filePath":"user-guide/mcp-tools/compare-runs.md"}'),n={name:"user-guide/mcp-tools/compare-runs.md"};function h(l,s,o,d,p,k){return a(),e("div",null,[...s[0]||(s[0]=[t("",15)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};