sparkforensics-server 0.2.2 → 0.2.4

package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js

@@ -1,5 +0,0 @@
-import{_ as a,a as t}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Executor Utilization","description":"","frontmatter":{"title":"Executor Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-utilization.md","filePath":"tuning-reference/bottleneck-utilization.md"}'),n={name:"tuning-reference/bottleneck-utilization.md"};function l(c,e,f,h,d,p){return s(),r("div",null,[...e[0]||(e[0]=[i(`<h1 id="bottleneck-utilization" tabindex="-1">Executor Utilization <a class="header-anchor" href="#bottleneck-utilization" aria-label="Permalink to &quot;Executor Utilization {#bottleneck-utilization}&quot;">​</a></h1><p><span class="tag">UTIL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Executor utilization measures how much of the cluster&#39;s allocated executor capacity a job actually keeps busy. When the average number of active executors trails the peak number allocated, the cluster is holding compute (cores and memory) that isn&#39;t running any tasks.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>The signal is the ratio of average active executors to the peak active executor count observed over the job&#39;s lifetime:</p><table tabindex="0"><thead><tr><th>avg active executors / peak</th><th>Level</th></tr></thead><tbody><tr><td>&lt; 60%</td><td>Info</td></tr><tr><td>&lt; 40%</td><td>Warning</td></tr><tr><td>&lt; 20%</td><td>Critical</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Idle executors are allocation you&#39;re paying for without getting work done. One common cause is too little task parallelism to occupy the allocated cores: the guidance is to keep at least as many partitions as there are cores across the executors, so no core sits idle<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Parallelism can also be lost by accident rather than by under-partitioning upfront: because <code>coalesce</code> is a narrow transformation, reducing partition count with it forces the <em>entire</em> upstream stage down to the reduced parallelism, not just the coalesce step, trading a shuffle for lost concurrency<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Separately, under dynamic allocation, a workload with many small tasks can end up over-provisioned: by default it requests enough executors to maximize parallelism for the task count, and with small tasks that can mean some executors &quot;might not even do any work,&quot; wasting resources on allocation overhead<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Reach for <code>repartition</code> (not <code>coalesce</code>) when you need to raise partition count or rebalance data: <code>coalesce</code> only avoids a shuffle when shrinking partition count, and doing so also drags the entire upstream stage down to the reduced parallelism<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</li><li>Size partitions so the task count is at least the core count available across executors, to avoid leaving cores idle<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</li><li>Under dynamic allocation, lower <code>spark.dynamicAllocation.executorAllocationRatio</code> below its default of <code>1.0</code> to scale back the number of executors requested for workloads made up of many small tasks<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li><li>Dynamic allocation requires shuffle tracking, the external shuffle service, or shuffle-block decommissioning to be enabled as a precondition<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>; without one of them, an executor removed mid-shuffle takes its unfetched shuffle files with it, forcing a recompute<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>, which undercuts using dynamic allocation to shed idle executors in the first place. <code>spark.dynamicAllocation.shuffleTracking.enabled</code> defaults to <code>true</code> since Spark 3.0, satisfying that precondition without needing a separate external shuffle service<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</li></ul><p>Restore parallelism after a heavy filter, and rein in over-provisioning for many-small-task jobs:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># repartition (not coalesce) rebalances and can raise partition count; aim &gt;= total executor cores</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.filter(heavy_predicate).repartition(spark.sparkContext.defaultParallelism)</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Scale back executors requested for many-small-task workloads (default ratio 1.0)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">spark.conf.set(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;spark.dynamicAllocation.executorAllocationRatio&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;0.5&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)  </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 0.5 is an example</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A low average-to-peak ratio isn&#39;t always waste. A bursty or I/O-bound job legitimately holds executors while tasks wait on external systems rather than burning cores, and the ratio is sensitive to short stages, where a brief spike in allocation skews the average without meaning the cluster was genuinely idle.</p><h2 id="caching-opportunity" tabindex="-1">Caching opportunity <a class="header-anchor" href="#caching-opportunity" aria-label="Permalink to &quot;Caching opportunity&quot;">​</a></h2><p><span class="tag">CACHE</span></p><p>When the same DataFrame, RDD, or input is scanned more than once, low utilization can trace back to repeated recomputation rather than idle cores. Spark keeps nothing between actions: transformations only build a DAG, and once an action finishes its intermediate results are discarded<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Call a second action on the same logic and Spark re-runs the whole DAG from the source, which can mean re-reading a terabyte from S3, re-reading Kafka, or repeating expensive decompression<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Fork that logic into two pipeline branches and you sign up to recompute everything twice<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>.</p><img class="light-only" src="`+a+'" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result."><img class="dark-only" src="'+t+'" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result."><p><a href="./caching.html"><code>cache()</code> and <code>persist()</code></a> are how you stop that. Persisting materializes the RDD (usually in memory on the executors) so it can be reused within the job, while Spark keeps its lineage to recompute any lost partition<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. For a dataset you read repeatedly, this is one of the most useful optimizations available: it parks a DataFrame, table, or RDD in temporary storage across the executors and makes later reads fast<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. Materialization is lazy and per-block, so only an action that touches every partition (<code>count()</code> or a full write) caches all of it; a subset scan like <code>take(10)</code> leaves a partial cache<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>.</p><p><code>cache()</code> is shorthand; <code>persist()</code> takes a <code>StorageLevel</code> and lets you pick the tradeoff:</p><table tabindex="0"><thead><tr><th>Level</th><th>Tradeoff</th></tr></thead><tbody><tr><td><code>MEMORY_ONLY</code></td><td>Fastest reads (deserialized JVM objects), but partitions that don&#39;t fit are recomputed on the fly each time<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</td></tr><tr><td><code>MEMORY_AND_DISK</code></td><td>Spills the overflow to disk instead of recomputing; Spark&#39;s default caching strategy and fine for most pipelines<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. For DataFrames, <code>.cache()</code> maps to this<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</td></tr><tr><td>Serialized (<code>_SER</code>)</td><td>Byte arrays instead of raw objects: smaller footprint, slower reads. Reach for it when memory is tight<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</td></tr><tr><td>Replicated (<code>_2</code>)</td><td>A second copy for fast fault recovery instead of waiting on recomputation, at twice the space<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup>.</td></tr></tbody></table><p>Whether to spill to disk hinges on recomputation cost: reading a block back from disk only beats recomputing it when the work that produced the data is expensive or filters out a large fraction, so the RDD guide says don&#39;t enable disk otherwise<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Caching is not free, and several cases don&#39;t warrant it:</p><ul><li><strong>Single use.</strong> Caching adds serialization, deserialization, and storage cost, so caching data you read once only slows you down<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup>.</li><li><strong>Data larger than storage memory,</strong> or a cheap transformation that isn&#39;t reused often regardless of size<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</li><li><strong>Memory pressure.</strong> Cache memory is memory taken from processing, and under the default spill strategy cached data can land on slower disk, so caching can cost more than just re-reading the source<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup>.</li><li><strong>Lost optimizer freedom.</strong> Once a dataset is cached, Catalyst works on the in-memory copy and can no longer push filters down to the source<sup class="footnote-ref"><a href="#fn9" id="fnref9:4">[9:4]</a></sup>.</li></ul><p>A persisted RDD you&#39;ve stopped using still occupies memory until the app ends or eviction forces it out, so call <code>unpersist()</code> to reclaim it deliberately, which matters most on shared clusters<sup class="footnote-ref"><a href="#fn9" id="fnref9:5">[9:5]</a></sup>.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Task parallelism:</strong> <a href="./partitioning.html">Partitioning</a></li><li><strong>Executor sizing:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 8 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Dynamic Resource Allocation</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 2 <a href="#fnref7" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-caching" target="_blank" rel="noreferrer">Spark Tips: Caching</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a> <a href="#fnref9:4" class="footnote-backref">↩︎</a> <a href="#fnref9:5" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide — Spark</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref11" class="footnote-backref">↩︎</a></p></li></ol></section>',31)])])}const g=o(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js

@@ -1 +0,0 @@
-import{_ as a,a as t}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Executor Utilization","description":"","frontmatter":{"title":"Executor Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-utilization.md","filePath":"tuning-reference/bottleneck-utilization.md"}'),n={name:"tuning-reference/bottleneck-utilization.md"};function l(c,e,f,h,d,p){return s(),r("div",null,[...e[0]||(e[0]=[i("",31)])])}const g=o(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as r}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Understanding findings","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/understanding-findings.md","filePath":"user-guide/understanding-findings.md"}'),i={name:"user-guide/understanding-findings.md"};function s(n,e,l,d,c,h){return t(),o("div",null,[...e[0]||(e[0]=[r('<h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to &quot;Understanding findings&quot;">​</a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>Turn on <a href="./getting-started.html#advanced-view">Advanced view</a> in the dashboard to see, per finding, the confidence level and a link to the same background reading collected here, opened in an in-app &quot;Reference&quot; panel next to the board. A few tags share that reading with another tag, because the underlying Spark-tuning material overlaps: <code>SFAIL</code> with <code>FAIL</code>, <code>PART</code> with <code>SHFL</code>, <code>SPEC</code> with <code>STRAG</code>, and <code>CACHE</code>/<code>LOCAL</code> with <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to &quot;Per-stage&quot;">​</a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to &quot;`SKEW`: Task skew {#skew}&quot;">​</a></h3><p>A small number of tasks take much longer than their peers in the same stage. For join-driven skew, enable AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>); otherwise salt the key or repartition on a better key.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to &quot;`SHFL`: Shuffle I/O {#shfl}&quot;">​</a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to &quot;`SPILL`: Memory and disk spill {#spill}&quot;">​</a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don&#39;t (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions).</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to &quot;`GC`: Garbage collection pressure {#gc}&quot;">​</a></h3><p>Tasks spend an unusually large share of time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to &quot;`FAIL`: Failed tasks {#fail}&quot;">​</a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors: check driver logs for the dominant failure reason.</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to &quot;`SFAIL`: Failed stage {#sfail}&quot;">​</a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to &quot;`STRAG`: Straggler tasks {#strag}&quot;">​</a></h3><p>A few tasks run much slower than the rest of their stage. Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that&#39;s a candidate for AQE&#39;s skew-join handling.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to &quot;`SPEC`: Speculation waste {#spec}&quot;">​</a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flags a confidence that scales with how far the wasted time sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to &quot;`RETRY`: Retry waste {#retry}&quot;">​</a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to &quot;`TINY`: Tiny tasks {#tiny}&quot;">​</a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to &quot;`PART`: Partition sizing {#part}&quot;">​</a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to &quot;`SLOW`: Stage slowness {#slow}&quot;">​</a></h3><p>A stage ran long overall without a more specific cause getting flagged. Often a partition-count problem: raise parallelism via <code>spark.sql.shuffle.partitions</code> or <code>spark.default.parallelism</code>, or check for a large per-task data volume driving heavy shuffle and spill.</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to &quot;`SHAPE`: Stage shape {#shape}&quot;">​</a></h3><p>The stage has an inefficient task count, output shape, or task-to-stage balance: for example, one straggler task taking a large fraction of the stage&#39;s wall-clock time.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to &quot;`HOST`: Slow host {#host}&quot;">​</a></h3><p>One executor is much slower than its peers. It may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable <code>spark.speculation</code> to relaunch a lagging task automatically.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to &quot;App-level&quot;">​</a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to &quot;`COLD`: Executor cold start {#cold}&quot;">​</a></h3><p>New executors take time to become available for work. Pre-warm the cluster, or use dynamic allocation.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to &quot;`UTIL`: Low utilization {#util}&quot;">​</a></h3><p>Allocated executors sit idle for a large share of the application run. Consider a smaller cluster, or enable dynamic allocation.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to &quot;`MEM`: Memory utilization {#mem}&quot;">​</a></h3><p>Executor memory or core capacity may be over- or under-provisioned. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can&#39;t be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant self-flags a confidence that scales with how far the estimated waste sits past a 1.5x buffer: it estimates waste from allocated-versus-used memory-time. Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to &quot;`CACHE`: Caching opportunity {#cache}&quot;">​</a></h3><p>A reusable dataset (re-read via the same SQL relation more than once) may be worth persisting between stages. Self-flags a confidence that scales with how many executions reuse the same relation: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to &quot;`CSTOR`: Cache storage {#cstor}&quot;">​</a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to &quot;`LOCAL`: Core usage locality {#local}&quot;">​</a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check <code>spark.locality.wait</code> settings and executor/data colocation. Self-flags a confidence that scales with the non-local ratio and sample size: the thresholds are our own noise floor for this metric.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to &quot;`CHRN`: Autoscaling churn {#chrn}&quot;">​</a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. Self-flags a confidence that scales with how far the short-lived-executor share sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to &quot;`JOBS`: Job failure rate {#jobs}&quot;">​</a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to &quot;`INCMP`: Incomplete run {#incmp}&quot;">​</a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to &quot;Configuration scope&quot;">​</a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to &quot;`CFG`: Configuration audit {#cfg}&quot;">​</a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage&#39;s behavior. Four properties are audited today:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won&#39;t survive executor removal.</li><li><code>spark.dynamicAllocation.maxExecutors</code>: flagged for inverted bounds or a missing upper bound.</li><li><code>spark.serializer</code>: flagged when still on the default Java serializer; <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below a safe floor.</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to &quot;SQL scope&quot;">​</a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to &quot;`PLAN`: Plan advisor {#plan}&quot;">​</a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan.</li><li>Small files: reading an excessive number of small files.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>.</li></ul>',59)])])}const f=a(i,[["render",s]]);export{p as __pageData,f as default};