sparkforensics-server 0.2.4 → 0.4.0

package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as s,o,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const b=JSON.parse('{"title":"Job Failure Rate","description":"","frontmatter":{"title":"Job Failure Rate"},"headers":[],"relativePath":"tuning-reference/bottleneck-job-failure-rate.md","filePath":"tuning-reference/bottleneck-job-failure-rate.md"}'),n={name:"tuning-reference/bottleneck-job-failure-rate.md"};function l(c,e,u,f,d,h){return o(),r("div",null,[...e[0]||(e[0]=[i("",21)])])}const k=s(n,[["render",l]]);export{b as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js

@@ -1,7 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Memory Utilization","description":"","frontmatter":{"title":"Memory Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-memory-utilization.md","filePath":"tuning-reference/bottleneck-memory-utilization.md"}'),r={name:"tuning-reference/bottleneck-memory-utilization.md"};function n(i,e,l,c,d,f){return t(),o("div",null,[...e[0]||(e[0]=[s(`<h1 id="bottleneck-memory-utilization" tabindex="-1">Memory Utilization <a class="header-anchor" href="#bottleneck-memory-utilization" aria-label="Permalink to &quot;Memory Utilization {#bottleneck-memory-utilization}&quot;">​</a></h1><p><span class="tag">MEM</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>An executor is a single JVM process that gets a fixed memory allocation when the application starts, and it holds that whole allocation for its entire lifetime, whether or not it has work to run<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Two things can leave that memory band poorly used: cores sitting idle inside a held executor, and a held executor whose memory drains of active work but is never released. This finding covers both.</p><p>An executor&#39;s core count is its concurrency ceiling. <code>spark.executor.cores</code> sets how many tasks it can run at once, so <code>--executor-cores 5</code> caps that executor at five concurrent tasks<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. The scheduler turns cores into task slots from <code>spark.executor.cores</code> and <code>spark.task.cpus</code> (minimum 1), with <code>spark.task.cpus</code> defaulting to one core per task<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Cores go idle whenever there are fewer runnable tasks than slots: a stage with fewer partitions than the total slots across executors leaves slots empty, and so does the tail of a stage where a <a href="./bottleneck-straggler.html">straggler</a> or two keep running after their peers finish. Raising <code>spark.task.cpus</code> above 1 has the same effect from the other side, since each task then reserves several cores and fewer run side by side. Through all of it the JVM keeps its fixed heap<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Rule</th><th>Signal</th><th>Status</th></tr></thead><tbody><tr><td>idleCores</td><td>Task parallelism below allocated cores while the executor is held</td><td>Validated</td></tr><tr><td>memoryBand</td><td>A whole executor&#39;s memory band held with little or no active work (dynamic-allocation idle timeouts)</td><td>Validated</td></tr><tr><td>wasteModel</td><td>Peak used memory well below allocated memory (unused headroom)</td><td>Experimental</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A held executor is allocation you pay for regardless of how busy it is. On YARN that is the memory YARN grants the container; on Kubernetes it is the pod memory limit<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. When cores idle or a whole executor lingers with nothing to do, that fixed band is reserved without returning work, so the cost lands whether or not tasks are running.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Keep executors from being oversized in the first place. The guidance is against packing every core of a node into one fat executor: assigning all 16 cores of a node to a single executor hurts HDFS throughput and drives excessive <a href="./bottleneck-gc.html">garbage collection</a>, so aim for a balance between tiny (one core per executor) and fat (one executor per node) sizing<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. On YARN, leave cores for the OS and Hadoop daemons instead of handing 100% of a node to Spark containers<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>For the held memory band, lean on <a href="./cluster-config.html">dynamic allocation</a> to reclaim executors once the work drains. It requests executors when tasks back up and frees them when they go idle<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Executors are added in rounds once tasks have been pending for <code>spark.dynamicAllocation.schedulerBacklogTimeout</code> (default 1s), then again every <code>spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</code> while the backlog holds<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. On the release side, an executor is removed after it has been idle longer than <code>spark.dynamicAllocation.executorIdleTimeout</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p>Cached data is the trap here. By default an executor holding <a href="./caching.html">cached blocks</a> is never removed, governed by <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code>, whose default is infinity<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. Such an executor keeps its full memory band indefinitely with no active tasks unless you set that timeout to a finite value, or turn on <code>spark.shuffle.service.fetch.rdd.enabled</code> so executors holding only disk-persisted blocks are treated as idle after <code>spark.dynamicAllocation.executorIdleTimeout</code> and released<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. To avoid holding barely-used executors at all, lower <code>spark.dynamicAllocation.executorAllocationRatio</code> (default 1.0, full parallelism) toward 0.5, since with small tasks full-parallelism allocation can request executors that never do any work<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Balanced executor sizing, not one fat executor per node</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.cores</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=5</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Reclaim idle executors; give cached holders a finite timeout</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.cachedExecutorIdleTimeout</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=300s</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.executorAllocationRatio</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.5</span></span></code></pre></div><h3 id="reading-the-wasted-memory-estimate-experimental" tabindex="-1">Reading the wasted-memory estimate <span class="tag">EXPERIMENTAL</span> <a class="header-anchor" href="#reading-the-wasted-memory-estimate-experimental" aria-label="Permalink to &quot;Reading the wasted-memory estimate &lt;span class=&quot;tag&quot;&gt;EXPERIMENTAL&lt;/span&gt;&quot;">​</a></h3><p>The wasteModel rule estimates unused (&quot;wasted&quot;) allocated memory from the gap between an executor&#39;s peak used memory and its allocated total. Treat it as a rough buffer heuristic, not a measurement. Peak used memory is a high-water mark rather than a ceiling: Spark records it under <code>peakMemoryMetrics.*</code>, and each figure is the maximum its pool ever reached, so peak sits at or below the allocated total by construction and the space between them is the executor&#39;s unused headroom<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Turning that headroom into a wasted number stays approximate for concrete reasons. The peak heap figure counts garbage, since <code>JVMHeapMemory</code> is the peak used heap including &quot;the amount of memory occupied by both live objects and garbage objects that have not been collected,&quot; so it overstates the live footprint<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. The region boundaries also move: <code>totalOnHeapStorageMemory</code> and <code>totalOffHeapStorageMemory</code> &quot;can vary over time, depending on the MemoryManager implementation,&quot; so there is no single fixed allocated-to-storage value to subtract a peak from<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. Peak metrics only reach the event log when <code>spark.eventLog.logStageExecutorMetrics</code> is true<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>, so without that setting there is nothing to estimate from. Use the number as a hint that an executor may be oversized, then confirm against sizing before acting on it.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The idleCores and memoryBand rules are validated: they rest on documented Spark concurrency and dynamic-allocation behavior. The wasteModel estimate is low-confidence and experimental. It is a buffer heuristic derived from peak-versus-allocated sampling, not an exact accounting of unused memory, so it should steer investigation rather than settle it.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Idle cores are not always waste. The tail of a stage legitimately leaves slots empty while a straggler or two finish<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>, so a snapshot of low parallelism can reflect a normal straggler tail rather than chronic under-utilization. The wasted-memory estimate depends on peak-memory sampling and is only as good as that sampling: the peak includes uncollected garbage, the managed-region boundaries shift under the MemoryManager, and the figures are absent entirely unless <code>spark.eventLog.logStageExecutorMetrics</code> is enabled<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</p><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark memory</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of executors, cores and memory for a Spark application</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Spark</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html" target="_blank" rel="noreferrer">Monitoring — Spark</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a></p></li></ol></section>`,22)])])}const u=a(r,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Memory Utilization","description":"","frontmatter":{"title":"Memory Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-memory-utilization.md","filePath":"tuning-reference/bottleneck-memory-utilization.md"}'),r={name:"tuning-reference/bottleneck-memory-utilization.md"};function n(i,e,l,c,d,f){return t(),o("div",null,[...e[0]||(e[0]=[s("",22)])])}const u=a(r,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as r,o,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Retry Waste","description":"","frontmatter":{"title":"Retry Waste"},"headers":[],"relativePath":"tuning-reference/bottleneck-retry-waste.md","filePath":"tuning-reference/bottleneck-retry-waste.md"}'),n={name:"tuning-reference/bottleneck-retry-waste.md"};function l(c,e,f,h,u,d){return o(),s("div",null,[...e[0]||(e[0]=[i('<h1 id="bottleneck-retry-waste" tabindex="-1">Retry Waste <a class="header-anchor" href="#bottleneck-retry-waste" aria-label="Permalink to &quot;Retry Waste {#bottleneck-retry-waste}&quot;">​</a></h1><p><span class="tag">RETRY</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>When a task attempt is superseded by a later retry, every bit of executor time the abandoned attempt spent counts for nothing. Every attempt, successful or not, accrues the standard task metrics: <code>executorRunTime</code> (&quot;elapsed time the executor spent running this task,&quot; including time fetching shuffle data) and <code>executorCpuTime</code><sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, but only the winning attempt&#39;s output survives. The TaskScheduler is what triggers the do-over: &quot;if an executor dies or a task throws an exception, the TaskScheduler resubmits the task to another executor, respecting Spark&#39;s task locality preferences&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Whatever the discarded attempt had already run is gone; none of it carries forward into the job&#39;s result.</p><img class="light-only" src="'+t+'" alt="Where a superseded retry sits in the escalation ladder, below the stage and application budgets that a job crosses only when the same failures keep recurring."><img class="dark-only" src="'+a+'" alt="Where a superseded retry sits in the escalation ladder, below the stage and application budgets that a job crosses only when the same failures keep recurring."><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>A stage&#39;s superseded task attempts are the signal: a later retry supersedes an earlier attempt after <a href="./bottleneck-failures.html">an executor is lost</a> (<code>ExecutorLostFailure</code>) or after a shuffle <code>FetchFailed</code>, since &quot;if an executor dies or a task throws an exception, the TaskScheduler resubmits the task to another executor, respecting Spark&#39;s task locality preferences&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. Because only the winning attempt&#39;s output survives, the abandoned attempt&#39;s <code>executorRunTime</code> (the elapsed executor time defined above) is the wasted time.</p><p>The two causes are handled differently by Spark&#39;s own machinery, which is how they can be told apart after the fact. An <code>ExecutorLostFailure</code> removes the executor (and any shuffle map output it had already written) from the pool outright<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. A shuffle <code>FetchFailed</code>, by contrast, is a read-side failure against a still-alive remote executor, and Spark&#39;s own exclusion machinery treats it as a distinct category from a general executor loss: <code>spark.excludeOnFailure.killExcludedExecutors</code> governs whether Spark kills executors &quot;excluded on fetch failure or excluded for the entire application&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>, a separate bucket from executor loss in Spark&#39;s accounting. That distinction (executor-loss bucket vs. fetch-failure bucket) is the basis for identifying which cause dominated a given stage&#39;s retry waste.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A lost executor can waste more than just the in-flight task&#39;s time: &quot;dynamic allocation may remove an executor before the shuffle completes, in which case the shuffle files written by that executor must be recomputed unnecessarily&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>, so prior map-output work from that same executor may need redoing too. A <code>FetchFailed</code> doesn&#39;t carry that same risk, since the remote executor stays alive and only the one failed read is lost<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Check the retried-attempt count and accumulated <code>executorRunTime</code>/<code>executorCpuTime</code> on the abandoned attempts directly, rather than trusting the stage&#39;s final wall-clock duration: a clean-looking stage can still be hiding significant wasted compute<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</li><li>If the dominant cause is executor loss, investigate node-level stability (OOM-kills, node death, aggressive dynamic-allocation deallocation) rather than the task logic itself, since the lost executor&#39;s prior shuffle-write work may also need recomputing<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</li><li>If the dominant cause is <code>FetchFailed</code>, check whether <code>spark.excludeOnFailure.killExcludedExecutors</code> is causing repeated exclusion churn on otherwise-live executors, and treat it separately from outright executor loss<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>.</li><li>See also <a href="./bottleneck-job-failure-rate.html">job failure rate</a>: retry waste can pile up quietly on a job that ultimately succeeds, while the same underlying causes (exhausted at a higher threshold) are what push a job to fail outright.</li></ul><blockquote><p><strong>PySpark:</strong> there&#39;s no attempt-level API to recover a discarded attempt&#39;s <code>executorRunTime</code>: pull it from the event log or history server UI&#39;s per-stage task list, filtering for tasks whose attempt number is greater than zero.</p></blockquote><p>The preventive lever: keep an executor&#39;s shuffle output available if it is lost, so retries don&#39;t recompute prior map work<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.shuffleTracking.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Some retry waste is unavoidable: transient cluster faults will always cost a few superseded attempts, and no tuning drives that to zero. The metric attributes burned executor time to the abandoned attempts, so its accuracy tracks the failure cause. When a lost executor also forces recompute of prior map output, the per-attempt <code>executorRunTime</code> undercounts the true cost; when an attempt fails almost immediately, it overcounts the compute actually lost. Read the number as a directional signal, not an exact ledger.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Shuffle recompute on executor loss:</strong> <a href="./shuffle.html">Shuffle</a></li><li><strong>Executor stability &amp; dynamic allocation:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-anatomy-of-spark-application" target="_blank" rel="noreferrer">Spark: Anatomy of Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling (Spark)</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li></ol></section>',22)])])}const g=r(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as r,o,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Retry Waste","description":"","frontmatter":{"title":"Retry Waste"},"headers":[],"relativePath":"tuning-reference/bottleneck-retry-waste.md","filePath":"tuning-reference/bottleneck-retry-waste.md"}'),n={name:"tuning-reference/bottleneck-retry-waste.md"};function l(c,e,f,h,u,d){return o(),s("div",null,[...e[0]||(e[0]=[i("",22)])])}const g=r(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Shuffle I/O","description":"","frontmatter":{"title":"Shuffle I/O"},"headers":[],"relativePath":"tuning-reference/bottleneck-shuffle.md","filePath":"tuning-reference/bottleneck-shuffle.md"}'),r={name:"tuning-reference/bottleneck-shuffle.md"};function i(n,e,f,l,c,d){return t(),s("div",null,[...e[0]||(e[0]=[o("",34)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js

@@ -1 +0,0 @@
-import{_ as e,o as s,c as t,a5 as i}from"./chunks/framework.DSg0KOwT.js";const f=JSON.parse('{"title":"Task Skew","description":"","frontmatter":{"title":"Task Skew"},"headers":[],"relativePath":"tuning-reference/bottleneck-skew.md","filePath":"tuning-reference/bottleneck-skew.md"}'),n={name:"tuning-reference/bottleneck-skew.md"};function o(r,a,l,h,p,d){return s(),t("div",null,[...a[0]||(a[0]=[i("",28)])])}const c=e(n,[["render",o]]);export{f as __pageData,c as default};

package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Slow Host","description":"","frontmatter":{"title":"Slow Host"},"headers":[],"relativePath":"tuning-reference/bottleneck-slow-host.md","filePath":"tuning-reference/bottleneck-slow-host.md"}'),i={name:"tuning-reference/bottleneck-slow-host.md"};function n(r,e,l,h,c,d){return a(),s("div",null,[...e[0]||(e[0]=[o("",27)])])}const u=t(i,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.js

@@ -1,5 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Small Files","description":"","frontmatter":{"title":"Small Files"},"headers":[],"relativePath":"tuning-reference/bottleneck-small-files.md","filePath":"tuning-reference/bottleneck-small-files.md"}'),o={name:"tuning-reference/bottleneck-small-files.md"};function n(r,e,l,f,h,c){return a(),s("div",null,[...e[0]||(e[0]=[i(`<h1 id="bottleneck-small-files" tabindex="-1">Small Files <a class="header-anchor" href="#bottleneck-small-files" aria-label="Permalink to &quot;Small Files {#bottleneck-small-files}&quot;">​</a></h1><p><span class="tag">SMALLFILES</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>The small-files problem is the cost of managing a large count of tiny files. Writing many small files runs up significant metadata overhead, and Spark handles that pattern badly; distributed filesystems such as HDFS handle it badly too, which is why it earned the name &quot;small file problem&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. The same trap shows up on the read side. Push <code>spark.sql.files.maxPartitionBytes</code> (default 128 MB, aligned to the <a href="./data-formats.html">Parquet</a> block size) too low and Spark carves the input into many small partition files, piling on disk I/O and the filesystem overhead of opening, closing, and listing directories, all of which are slow on a distributed store<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><p>Spark writes one file per output partition, so a job left with 200 partitions writes 200 files and one left with 3000 writes 3000, even when many of those files hold only a handful of rows. That count then punishes every downstream job forced to read thousands of tiny files<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The opposite extreme is not free either: files that are too large make it inefficient to read a whole block when you only need a few rows<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Each SQL plan node carries file-count and file-size metrics Spark reports directly: <code>number of files read</code>/<code>size of files read</code> on the read side, <code>number of written files</code>/<code>written output</code> on the write side. A node reads as a small-files problem once its file count on a given side is high and the resulting average file size is small.</p><table tabindex="0"><thead><tr><th>Signal (per plan node, checked separately for read and write)</th><th>Fires when</th></tr></thead><tbody><tr><td>File count</td><td>&gt; 100</td></tr><tr><td>Average file size (bytes ÷ file count)</td><td>&lt; 3 MB</td></tr></tbody></table><p>Both conditions have to hold together, so a node with thousands of files that are each big enough, or a handful of genuinely tiny ones, doesn&#39;t register. The signal comes entirely from those file-count and file-size metrics, independent of <code>spark.sql.files.maxPartitionBytes</code>.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Every extra file is another open, close, and directory-list operation, and on a distributed filesystem those are not cheap<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. The count compounds downstream: a stage that scatters 3000 tiny files hands the next job 3000 files to read back, so the metadata tax is paid twice over<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. None of it is a single dramatic failure. The waste is spread thin across the file count, which is exactly why it goes unnoticed until listing and I/O start to dominate.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Repartition right before the write so the output lands in fewer, better-sized files.</p><ul><li><code>coalesce()</code> is a shuffle-free merge: it fuses existing partitions with no data movement, so <code>df.coalesce(100).write.parquet(...)</code> collapses 3000 tiny files into 100 reasonable ones<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. It does not rebalance, though, so uneven inputs stay uneven once lumped together, and pushing it to <code>coalesce(1)</code> kills parallelism by forcing one executor to do all the work<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>.</li><li><code>repartition()</code> is a full reshuffle that buys even distribution at the cost of that <a href="./shuffle.html">shuffle</a>. Reach for it when the distribution itself needs fixing, not just the file count<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> both are one-line calls on a DataFrame: <code>df.coalesce(100)</code> for a shuffle-free merge, or <code>df.repartition(100)</code> when the data also needs rebalancing.</p></blockquote><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Collapse many tiny output files without a shuffle (distribution already even)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).write.parquet(path)</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Full reshuffle to a target count when the distribution itself needs fixing</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df.repartition(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).write.parquet(path)</span></span></code></pre></div><p>For runtime control, <a href="./aqe.html">Adaptive Query Execution</a> can merge tiny shuffle partitions on its own. With <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> (default true) set, AQE coalesces contiguous shuffle partitions toward <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code> (default 64 MB)<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>. Just mind the scope: AQE only engages after the first shuffle, so it fixes shuffle-output partition counts but will not repair input-side partitioning or a bad file layout on disk<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The core inference, that one output file per partition times a high partition count yields many tiny files, is grounded directly in Spark&#39;s write behavior<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>, and the mitigation levers (<code>coalesce()</code>, <code>repartition()</code>, and AQE coalescing) are documented Spark features<sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A high file count can be perfectly legitimate: partitioned output deliberately fans data across many files by partition column, so a large count there is by design, not a defect. And AQE coalescing is not a cure-all here, since it only affects post-shuffle partitions and leaves the input file layout untouched<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. The signal reflects the shape, not intent, so confirm the write is not an intended partitioned layout before acting.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Partition sizing:</strong> <a href="./partitioning.html">Partitioning</a></li><li><strong>Too many small tasks:</strong> <a href="./bottleneck-tiny-tasks.html">Tiny Tasks</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li></ol></section>`,25)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Small Files","description":"","frontmatter":{"title":"Small Files"},"headers":[],"relativePath":"tuning-reference/bottleneck-small-files.md","filePath":"tuning-reference/bottleneck-small-files.md"}'),o={name:"tuning-reference/bottleneck-small-files.md"};function n(r,e,l,f,h,c){return a(),s("div",null,[...e[0]||(e[0]=[i("",25)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js

@@ -1,6 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const i="/docs/assets/spill-classification.BU2euYDO.svg",r="/docs/assets/spill-classification.dark.D7i1M40d.svg",u=JSON.parse('{"title":"Memory / Disk Spill","description":"","frontmatter":{"title":"Memory / Disk Spill"},"headers":[],"relativePath":"tuning-reference/bottleneck-spill.md","filePath":"tuning-reference/bottleneck-spill.md"}'),n={name:"tuning-reference/bottleneck-spill.md"};function l(c,e,f,h,p,d){return t(),o("div",null,[...e[0]||(e[0]=[s('<h1 id="bottleneck-spill" tabindex="-1">Memory / Disk Spill <a class="header-anchor" href="#bottleneck-spill" aria-label="Permalink to &quot;Memory / Disk Spill {#bottleneck-spill}&quot;">​</a></h1><p><span class="tag">SPILL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Spill happens when a task&#39;s share of execution memory runs out during a sort or hash aggregation. Each task gets its own <code>TaskMemoryManager</code>, which arbitrates the shared execution pool across every task running concurrently on an executor; with <code>n</code> tasks running concurrently, each is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory: a soft limit, not a hard stop.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> When a task&#39;s sort or hash-aggregation operator keeps requesting execution-memory pages it can&#39;t get, Spark doesn&#39;t fail immediately: it blocks the requesting task, triggers a spill of that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><p>The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> spill to the executors&#39; local disks, at the location set by <code>spark.local.directory</code>.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> SQL physical operators apply the same idea with their own guardrails: <code>SortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row-count threshold, which by default is <code>spark.shuffle.spill.numElementsForceSpillThreshold</code>.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup> Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Detection: any <code>memoryBytesSpilled &gt; 0</code> → Warning. The classification is the actionable signal.</p><p>Spill classification:</p><ul><li><code>skew</code>: ≥ 80% of tasks have zero spill; fix: address <a href="./bottleneck-skew.html">task skew</a>, not memory.</li><li><code>volume</code>: &lt; 20% of tasks have zero spill; fix: more partitions or more memory.</li><li><code>unclassified</code>: neither condition; treat as volume.</li></ul><img class="light-only" src="'+i+'" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><img class="dark-only" src="'+r+`" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Spill is the fallback Spark reaches for once a task can no longer get the execution memory it&#39;s asking for: rather than failing outright, it blocks the task, writes its buffered data to disk, or (if that&#39;s not enough) throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup> A task that&#39;s spilling has already given up pure in-memory speed to keep running at all, which is why the skew-vs-volume split above is the actionable part of the signal: the two failure shapes call for opposite fixes.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li><strong>Skew</strong>: When only a few tasks spill, the problem is the shape of the data, not the size of the memory pool. <code>repartition()</code> is the tool for this: reach for it specifically to fix a lopsided partition distribution or to increase partition count<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>, in contrast to <code>coalesce()</code>, which never rebalances skewed data because it just stacks existing partitions together: partitions that were uneven going in are still uneven coming out.<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> For skewed joins, <a href="./aqe.html">Adaptive Query Execution</a> can detect and split<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup> oversized partitions automatically: a partition counts as skewed once it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size, and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>).<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> Before AQE existed, the manual equivalent was salting: adding a prefix to the skewed keys to make the same key look different, then adjusting the data distribution accordingly.<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> Databricks&#39; <code>/*+ SKEW(...) */</code> hint lets you name the skewed relation and specific key values directly, so the planner targets just those keys instead of relying on automatic detection.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Volume</strong>: When most tasks spill, the fix is more room to work with: more partitions, more memory, or both. <code>repartition(n)</code> guarantees exactly <code>n</code> output partitions via a hash shuffle<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>, and since Spark&#39;s per-task startup overhead is low (unlike MapReduce&#39;s), the general bias is to err toward more partitions rather than fewer.<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup><code>spark.sql.files.maxPartitionBytes</code> (default <code>128 MB</code>) is the equivalent lever on the input side, governing how much file data gets packed into each input partition before a shuffle even happens.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup> On the memory side, execution memory is a fraction of the JVM heap. <code>spark.memory.fraction</code> (default <code>0.6</code>) sizes the shared execution/storage region as <code>(JVM heap − 300 MiB) × spark.memory.fraction</code><sup class="footnote-ref"><a href="#fn13" id="fnref13">[13]</a></sup>. Because each task&#39;s slice of that region is capped by the <code>TaskMemoryManager</code> at between <code>1/(2n)</code> and <code>1/n</code> of the total,<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup> giving executors more memory, or running fewer concurrent tasks on each one, directly raises the ceiling before spill kicks in.</li></ul><p>For the <strong>volume</strong> case, give tasks more room: defaults shown, comments say which way to move:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Create more, smaller input partitions before the shuffle (default 128m)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.sql.files.maxPartitionBytes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=128m</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Execution/storage share of (JVM heap - 300MiB) (default 0.6);</span></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># raising it grows execution memory but starves the untracked user-memory region</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.memory.fraction</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.6</span></span></code></pre></div><p>For the <strong>skew</strong> case (only a few tasks spill), the fix is <code>df.repartition(n)</code>, not more memory: see <a href="./partitioning.html">Partitioning</a>.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The core detection (any <code>memoryBytesSpilled &gt; 0</code>) and the skew-vs-volume classification are validated: both the skew branch (most tasks spill nothing, so the shape of the data is the problem) and the volume branch (nearly every task spills, so the pool is too small) map to a well-understood fix, and neither needs further validation. <span class="tag">EXPERIMENTAL</span> When a spill matches neither shape, it falls back to a low-confidence unclassified finding that still requires validation, because there is no confirmed cause to act on; the page defaults it to the volume remedy, but that is a guess, not a diagnosis.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Some spill is normal. A large aggregation or join can spill by design once its working set outgrows execution memory, and a small spill on a healthy stage rarely repays the effort of chasing it. The raw <code>memoryBytesSpilled &gt; 0</code> trigger fires on both the actionable and the routine cases, so treat a small spill as informational until the skew-vs-volume split says otherwise.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Why it happens:</strong> <a href="./memory-model.html">Memory Management</a>, <a href="./partitioning.html">Partitioning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Diving into Spark Memory Management</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join in SQL Adaptive Execution</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref9" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join Optimization</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2): Cloudera Engineering Blog</a> <a href="#fnref12" class="footnote-backref">↩︎</a></p></li><li id="fn13" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref13" class="footnote-backref">↩︎</a></p></li></ol></section>`,26)])])}const k=a(n,[["render",l]]);export{u as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const i="/docs/assets/spill-classification.BU2euYDO.svg",r="/docs/assets/spill-classification.dark.D7i1M40d.svg",u=JSON.parse('{"title":"Memory / Disk Spill","description":"","frontmatter":{"title":"Memory / Disk Spill"},"headers":[],"relativePath":"tuning-reference/bottleneck-spill.md","filePath":"tuning-reference/bottleneck-spill.md"}'),n={name:"tuning-reference/bottleneck-spill.md"};function l(c,e,f,h,p,d){return t(),o("div",null,[...e[0]||(e[0]=[s("",26)])])}const k=a(n,[["render",l]]);export{u as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js

@@ -1,7 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Stragglers","description":"","frontmatter":{"title":"Stragglers"},"headers":[],"relativePath":"tuning-reference/bottleneck-straggler.md","filePath":"tuning-reference/bottleneck-straggler.md"}'),i={name:"tuning-reference/bottleneck-straggler.md"};function r(n,e,l,h,c,d){return a(),s("div",null,[...e[0]||(e[0]=[o(`<h1 id="bottleneck-straggler" tabindex="-1">Stragglers <a class="header-anchor" href="#bottleneck-straggler" aria-label="Permalink to &quot;Stragglers {#bottleneck-straggler}&quot;">​</a></h1><p><span class="tag">STRAG</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A straggler is one task (or a small handful of tasks) that runs far longer than the rest of the tasks in its stage, even when the stage is otherwise healthy. Because a stage only completes once its last task finishes, a single straggler holds up the whole stage, and every other executor sits idle waiting for it to catch up.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>A task counts as a straggler when speculative tasks fired for it, or when more than 5% of the tasks in a stage run at least 4× the stage&#39;s median task duration; this rule only applies once a stage has at least 10 tasks.</p><p>&quot;Duration&quot; here is <code>executorRunTime</code>: elapsed wall-clock time, not CPU time, and it already includes any time the task spent blocked fetching shuffle data<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. That matters when tracking down why a task is slow: a long <code>executorRunTime</code> doesn&#39;t necessarily mean more compute happened. A <a href="./bottleneck-gc.html">garbage-collection pause</a> is counted inside it rather than added on top (<code>jvmGCTime</code> is a subset of <code>executorRunTime</code>, not additive<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>), and a task waiting on a remote shuffle block it needs next shows up in <code>shuffleReadMetrics.fetchWaitTime</code>, which only counts genuine blocking time, not blocks being prefetched in the background<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>The &quot;speculative tasks fired&quot; half of the rule is Spark&#39;s own detector: once <code>spark.speculation.quantile</code> (default <code>0.9</code>) of a stage&#39;s tasks finish, Spark compares each remaining task&#39;s duration against <code>spark.speculation.multiplier</code> (default <code>3</code>) times the median of the tasks that already finished, subject to a <code>spark.speculation.minTaskRuntime</code> floor (default <code>100ms</code>) so short tasks aren&#39;t flagged just for being slower than a tiny median<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Speculation itself is off by default (<code>spark.speculation</code> defaults to <code>false</code>), so this half of the rule only fires on stages where it&#39;s been turned on<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>Watch for overlap with <a href="./bottleneck-skew.html">task skew</a>: an unevenly distributed key sends one partition far more data than its peers, and that partition&#39;s task will trip this same duration threshold even though the underlying cause is data volume, not a slow host or a GC pause. Check the skew signals before assuming the latter.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A straggler wastes cluster capacity the same way a skewed stage does: every executor other than the one running the slow task finishes early and idles, while total job time still tracks the single slowest task. Because a straggler&#39;s duration can be inflated by a GC pause<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup> or a slow <a href="./shuffle.html">shuffle</a> fetch<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup> rather than genuinely more work, it&#39;s worth checking those angles (and whether the real cause is skew rather than anything task-local) before assuming a hardware explanation.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Enable speculative execution: <code>spark.speculation</code> is <code>false</code> by default, so nothing reruns a slow task automatically until it&#39;s turned on<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>.</li><li>Tune the trigger: <code>spark.speculation.quantile</code> (default <code>0.9</code>) sets how much of the stage must finish before speculation kicks in, and <code>spark.speculation.multiplier</code> (default <code>3</code>) sets how many times slower than the median a task must be. For stages with very few tasks, <code>spark.speculation.task.duration.threshold</code> (available since 3.0.0) gives an absolute-duration trigger instead of relying on the median<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</li><li>Since Spark 3.4, <code>spark.speculation.efficiency.enabled</code> (default <code>true</code>) adds a filter so a task is only speculated if its data-process rate is below the stage average (times <code>spark.speculation.efficiency.processRateMultiplier</code>, default <code>0.75</code>) or its duration exceeds <code>spark.speculation.efficiency.longRunTaskFactor</code> (default <code>2</code>) times the same time threshold. This avoids wasting a duplicate task slot on work that&#39;s simply doing more, not running slower<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</li><li>If the cause is really a skewed key, treat it as <a href="./bottleneck-skew.html">task skew</a> instead: AQE&#39;s skew-join optimization automatically splits a partition once it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size and above <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Manual salting (appending a random prefix to the skewed key so it spreads across more partitions) is the pre-AQE fallback, though the design doc behind AQE&#39;s skew handling calls salting and other manual approaches limited compared to the automatic option<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> speculation settings can be set on the session directly, no <code>spark-submit</code> flag needed: <code>spark.conf.set(&quot;spark.speculation&quot;, &quot;true&quot;)</code>, then the quantile/multiplier equivalents the same way.</p></blockquote><p>Speculation config: defaults shown, plus the absolute-duration trigger for small stages:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Speculatively relaunch straggler tasks (OFF by default)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.quantile</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.9          </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; portion of tasks finished before speculation begins</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.multiplier</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=3          </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; multiple of median duration that marks a task slow</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Absolute-duration trigger for stages with very few tasks (since Spark 3.0)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.task.duration.threshold</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=10s   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 10s is an example</span></span></code></pre></div><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>Validated.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>The 4x-median rule flags a slow task, but slow is not the same as broken. The same threshold trips on a skewed key that simply has more data to process, or on a task that spent its time in a GC pause rather than doing extra work, so a flagged task is not automatically a slow host. Small stages make this worse: with only a handful of tasks the median is unstable, and one moderately slow task can look like a straggler against a median computed from too few peers.</p><h2 id="bottleneck-speculation-waste" tabindex="-1">Speculation waste <a class="header-anchor" href="#bottleneck-speculation-waste" aria-label="Permalink to &quot;Speculation waste {#bottleneck-speculation-waste}&quot;">​</a></h2><p><span class="tag">SPEC</span></p><p>A straggler is reported by whichever signal, straggler share or speculative-task count, best explains it. Speculation waste is a separate, narrower signal: the executor time speculative attempts burned without confirming a genuine straggler, whether the speculative copy lost the race to the original attempt or the other way around. Either way, the losing attempt&#39;s executor time is pure waste.</p><h3 id="how-it-s-detected-1" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected-1" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Signal</th><th>Fires when</th></tr></thead><tbody><tr><td>Wasted speculative attempts in a stage</td><td>≥ 5</td></tr><tr><td>Wasted executor time from those attempts</td><td>≥ 60 seconds</td></tr></tbody></table><p>Both conditions have to hold together. Severity tracks the estimated recoverable time as a share of the app&#39;s total runtime, the same model used across this page: ≥2% is critical, ≥0.5% is warning, anything smaller is info.</p><h3 id="why-it-matters-1" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters-1" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>Every wasted speculative attempt occupies an executor slot that could have run other work, so a stage generating a lot of speculative waste is trading cluster capacity for copies that never pay off.</p><h3 id="how-to-fix-it-1" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it-1" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><p>If task durations are naturally variable rather than genuine stragglers, speculation is firing too eagerly: tune <code>spark.speculation.multiplier</code> (require a bigger gap from the median before speculating) or <code>spark.speculation.quantile</code> (wait for more of the stage to finish first) so fewer ordinary slow tasks get speculated in the first place.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>When the real cause is a skewed key:</strong> <a href="./partitioning.html">Partitioning</a>, <a href="./aqe.html">Adaptive Query Execution</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html#optimizing-skew-join" target="_blank" rel="noreferrer">Optimizing Skew Join (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join at Runtime with New Adaptive Execution</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li></ol></section>`,34)])])}const u=t(i,[["render",r]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Stragglers","description":"","frontmatter":{"title":"Stragglers"},"headers":[],"relativePath":"tuning-reference/bottleneck-straggler.md","filePath":"tuning-reference/bottleneck-straggler.md"}'),i={name:"tuning-reference/bottleneck-straggler.md"};function r(n,e,l,h,c,d){return a(),s("div",null,[...e[0]||(e[0]=[o("",34)])])}const u=t(i,[["render",r]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js

@@ -1,8 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Tiny Tasks","description":"","frontmatter":{"title":"Tiny Tasks"},"headers":[],"relativePath":"tuning-reference/bottleneck-tiny-tasks.md","filePath":"tuning-reference/bottleneck-tiny-tasks.md"}'),o={name:"tuning-reference/bottleneck-tiny-tasks.md"};function n(r,e,l,h,f,d){return a(),s("div",null,[...e[0]||(e[0]=[i(`<h1 id="bottleneck-tiny-tasks" tabindex="-1">Tiny Tasks <a class="header-anchor" href="#bottleneck-tiny-tasks" aria-label="Permalink to &quot;Tiny Tasks {#bottleneck-tiny-tasks}&quot;">​</a></h1><p><span class="tag">TINY</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Every task pays a fixed placement and serialization cost before it does any real work. Spark schedules around data locality rather than moving data to code: &quot;Spark builds its scheduling around this general principle&quot; that shipping serialized code is cheaper than shipping data, preferring <code>PROCESS_LOCAL</code> locality down to <code>ANY</code> and waiting a configurable timeout for a busy CPU to free up before shipping data to a farther executor<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. On top of that placement cost, every task also pays a serialization cost for its closure and, if not using Kryo, for its data: Java&#39;s default <code>ObjectOutputStream</code>-based serialization &quot;is flexible but often quite slow, and leads to large serialized formats,&quot; while Kryo is &quot;significantly faster and more compact than Java serialization (often as much as 10x)&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. None of that overhead is large per task: Spark&#39;s own guidance leans toward more, smaller tasks rather than fewer, larger ones, unlike MapReduce: &quot;it&#39;s almost always better to err on the side of a larger number of tasks (and thus partitions)... The difference stems from the fact that MapReduce has a high startup overhead for tasks, while Spark does not&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. But once partitions get small enough, that per-task overhead stops being negligible and starts to dominate.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>The failure mode is partitions so small that scheduling and serialization overhead outweighs the actual work. With Spark&#39;s default of 200 shuffle partitions applied to a dataset of only a few megabytes, &quot;those 200 partitions will each get like ten rows. Tasks become microscopic. Most of your CPUs will just be sitting there doing nothing, wasting cluster hours&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. The same small-partition penalty shows up on the <a href="./bottleneck-shuffle.html">shuffle-read side</a>: production measurements found &quot;the average shuffle block size is only 10s of KBs, which leads to delayed shuffle data fetch,&quot; and the shuffle reduce stages with the largest fetch delays (over 30 seconds per task) were consistently the ones with small block sizes<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. A high task count paired with very short median task duration, and/or very small shuffle block sizes on the read side, are the signals to look for.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Tiny tasks waste cluster capacity even though no single task is a straggler the way <a href="./bottleneck-skew.html">skewed</a> or <a href="./bottleneck-straggler.html">straggler</a> tasks are: the cost here is spread evenly across thousands of tasks, each individually cheap but collectively adding up in scheduling and serialization overhead, while cores that could be doing other useful work sit mostly idle between task launches<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li><code>coalesce()</code> merges existing partitions without a shuffle (&quot;no data movement, no shuffle&quot;) and is typically used right before a write, e.g. <code>df.coalesce(100).write.parquet(...)</code>, to collapse many tiny output files into a manageable number cheaply<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. Because it only stacks partitions together rather than redistributing data, it doesn&#39;t fix an uneven underlying distribution (uneven input partitions stay uneven, just grouped) and pushing it too far (<code>coalesce(1)</code>) kills parallelism by funneling all work onto one executor while the rest sit idle<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</li><li><code>repartition()</code> performs a full reshuffle, giving control and rebalancing that Spark won&#39;t do on its own: with <code>df.repartition(200)</code>, &quot;you&#39;re paying for predictability&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Use it when the partitioning itself needs to be fixed or rebalanced, not merely reduced in count, accepting the shuffle cost to get there.</li><li>There&#39;s a middle option, <code>coalesce(N, shuffle=True)</code>, which &quot;acts more like a repartition, but leaning toward reduction... not free (you pay for the shuffle cost) but you get better distribution and fewer partitions&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</li><li>Rule of thumb: reach for <code>coalesce()</code> when only the partition count needs to shrink and the existing distribution is already reasonably even (e.g. collapsing output files); reach for <code>repartition()</code> when the distribution itself is the problem.</li></ul><blockquote><p><strong>PySpark:</strong> both are one-line calls on a DataFrame: <code>df.coalesce(100)</code> for a shuffle-free merge, <code>df.repartition(200)</code> for a full reshuffle, or <code>df.coalesce(100, shuffle=True)</code> for the shuffled middle option.</p></blockquote><p>The three sizing calls, side by side:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Collapse many tiny output files without a shuffle (use when distribution is already even)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).write.parquet(path)</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Full reshuffle to a target count — use when the distribution itself needs rebalancing</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.repartition(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">200</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Middle ground: reduce partition count but still rebalance (pays a shuffle)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">shuffle</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Very short tasks are not always a problem. A tiny final stage can be perfectly fine, and one that just writes a small result set doesn&#39;t need to be resized. The scheduling-overhead penalty only bites when tiny tasks dominate the stage, so treat a handful of short tasks as noise rather than a finding.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Partition sizing:</strong> <a href="./partitioning.html">Partitioning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning (Spark)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2): Cloudera</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-30602" target="_blank" rel="noreferrer">SPARK-30602</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li></ol></section>`,19)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Tiny Tasks","description":"","frontmatter":{"title":"Tiny Tasks"},"headers":[],"relativePath":"tuning-reference/bottleneck-tiny-tasks.md","filePath":"tuning-reference/bottleneck-tiny-tasks.md"}'),o={name:"tuning-reference/bottleneck-tiny-tasks.md"};function n(r,e,l,h,f,d){return a(),s("div",null,[...e[0]||(e[0]=[i("",19)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Executor Utilization","description":"","frontmatter":{"title":"Executor Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-utilization.md","filePath":"tuning-reference/bottleneck-utilization.md"}'),n={name:"tuning-reference/bottleneck-utilization.md"};function l(c,e,f,h,d,p){return s(),r("div",null,[...e[0]||(e[0]=[i("",44)])])}const g=o(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js

@@ -1 +0,0 @@
-import{_ as a,o,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/cache-lifecycle.rEOVYQNU.svg",n="/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg",m=JSON.parse('{"title":"Caching & Persistence","description":"","frontmatter":{"title":"Caching & Persistence"},"headers":[],"relativePath":"tuning-reference/caching.md","filePath":"tuning-reference/caching.md"}'),f={name:"tuning-reference/caching.md"};function c(i,e,l,h,d,p){return o(),t("div",null,[...e[0]||(e[0]=[r('<h1 id="caching" tabindex="-1">Caching &amp; Persistence <a class="header-anchor" href="#caching" aria-label="Permalink to &quot;Caching &amp; Persistence {#caching}&quot;">​</a></h1><h2 id="how-persisting-and-checkpointing-differ" tabindex="-1">How persisting and checkpointing differ <a class="header-anchor" href="#how-persisting-and-checkpointing-differ" aria-label="Permalink to &quot;How persisting and checkpointing differ&quot;">​</a></h2><p>Spark&#39;s <code>persist()</code> is the general mechanism for keeping a DataFrame&#39;s already-computed partitions around instead of recomputing them from source on every action; <code>cache()</code> is shorthand for <code>persist(StorageLevel.MEMORY_AND_DISK)</code>: it calls persist with that one fixed level internally<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, whereas persist() lets you choose any storage level (in-memory vs on-disk, serialized vs deserialized, replicated or not)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. When the level passed to persist() is exactly <code>MEMORY_AND_DISK</code>, the two calls are identical<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</p><p>Each storage level is defined by five attributes: <code>useDisk</code>, <code>useMemory</code>, <code>useOffHeap</code>, <code>deserialized</code>, and <code>replication</code><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Under <code>MEMORY_AND_DISK</code>, Spark stores data directly as objects in memory and serializes only the portion that doesn&#39;t fit, writing that overflow to disk<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. <code>MEMORY_AND_DISK_SER</code> behaves the same way except the data kept in memory is also serialized (data written to disk is always serialized under either level)<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Serialized byte streams use less memory than deserialized JVM objects, which carry structural overhead, but reading them back costs more CPU than reading deserialized objects directly<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p><p>Checkpointing is a related but distinct mechanism. Instead of persisting, it writes an RDD&#39;s partitions to an external, reliable store (HDFS, S3) and drops the lineage, the dependency chain Spark would otherwise use to recompute it<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. That truncation doesn&#39;t happen the instant <code>.checkpoint()</code> is called; the call only marks the RDD, and the actual truncation runs later, in <code>doCheckpoint()</code>, after a job using the RDD completes. At that point the RDD is already materialized, and its dependencies and old parents are cleared<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. A related call, <code>localCheckpoint()</code>, also truncates lineage using Spark&#39;s caching layer, but trades fault tolerance for speed: its data lives in ephemeral local executor storage rather than a reliable filesystem, so losing an executor mid-computation can make that data permanently unrecoverable<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>.</p><h2 id="when-caching-pays-off" tabindex="-1">When caching pays off <a class="header-anchor" href="#when-caching-pays-off" aria-label="Permalink to &quot;When caching pays off&quot;">​</a></h2><p>Because of how that materialization works, caching earns its keep only in specific situations. It&#39;s worth reaching for when the same DataFrame is scanned or recomputed more than once downstream: repeated queries against the same base data, iterative ML training loops, or interactive exploration where a dataset feeds multiple branches<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. In one book benchmark, caching a 10M-row DataFrame and materializing it with <code>count()</code> cut a subsequent <code>count()</code> from 5.11s to 0.44s, roughly a 12x speedup<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</p><p>A few signals point the other way, toward caching having no effect or actively hurting:</p><ul><li><strong>Single-use data.</strong> If a dataset is processed only once downstream, caching just adds serialization and storage-bookkeeping cost for no reuse benefit<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>.</li><li><strong>Data too big to fit.</strong> Caching is all-or-nothing per partition: a DataFrame can be &quot;fractionally cached&quot; across its partitions, but individual partitions can&#39;t be split. With room for only 4.5 of 8 partitions, exactly 4 get cached; the uncached remainder is recomputed on every access, which can end up slower than not caching at all<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup>.</li><li><strong>Partial materialization.</strong> <code>cache()</code>/<code>persist()</code> are lazy: nothing is cached until an action forces a full pass. <code>count()</code> forces a genuine full pass and materializes every partition, but an action like <code>take(1)</code> computes and caches only the one partition Catalyst needs, so a later full scan still recomputes most of the data. The same applies to <code>take(10)</code>, <code>limit(100)</code>, or any other partial or filtered scan: only the blocks Spark was forced to touch get cached, and the rest stays lazily uncomputed<sup class="footnote-ref"><a href="#fn2" id="fnref2:6">[2:6]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</li><li><strong>Plan mismatch.</strong> Caching wraps the <em>analyzed</em> (pre-optimization) logical plan in an <code>InMemoryRelation</code>. A logically-equivalent query written differently produces a different analyzed plan, so Spark can silently miss the cache and recompute from scratch even though the optimized plan would have been identical<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</li><li><strong>Optimizer limitations.</strong> Caching freezes Catalyst&#39;s optimization opportunities at the cached point. It can, for example, block <a href="./data-formats.html">predicate pushdown</a> once data is served from the in-memory cache instead of the source<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</li></ul><h2 id="why-cached-blocks-don-t-stick-around" tabindex="-1">Why cached blocks don&#39;t stick around <a class="header-anchor" href="#why-cached-blocks-don-t-stick-around" aria-label="Permalink to &quot;Why cached blocks don&#39;t stick around&quot;">​</a></h2><p>Materializing a cache is only half the story, though. A <code>cache()</code> followed by <code>count()</code> guarantees a materialization pass over every partition at that moment: caching happens locally and incrementally, with each executor&#39;s BlockManager storing only the blocks it computes, as it computes them, and no centralized &quot;cache the whole dataset&quot; step<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>. It does not guarantee those partitions stay cached afterward. Two things can undo it:</p><ul><li><strong>Executor loss.</strong> If a block was cached on an executor that later goes away, the cached data goes with it, unless a replicated storage level such as <code>MEMORY_AND_DISK_2</code> was used<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. This matters specifically for <a href="./cluster-config.html">dynamic allocation</a>: Spark&#39;s documentation is explicit that caching together with dynamic allocation &quot;is NOT safe,&quot; because reclaiming idle executors takes their cached blocks with them<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>.</li><li><strong>Memory pressure eviction.</strong> Cached blocks can be evicted by other operations&#39; memory demands after materialization, independent of any executor loss<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>.</li></ul><p>Eviction itself is governed by LRU: Spark removes the least-recently-used cached blocks when storage memory is under pressure<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup><sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. The trigger is memory contention between Spark&#39;s <a href="./memory-model.html">unified Execution and Storage regions</a>: under the <code>UnifiedMemoryManager</code>, Execution has priority, so if a task needs execution memory for a shuffle, join, aggregation, or sort and Storage is occupying that space, Spark evicts cached blocks to free it, and this can happen at any time, not only the next time the cache is accessed<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup>. What happens to an evicted block depends on its storage level, not on a separate spill decision: a disk-backed level (<code>MEMORY_AND_DISK</code>, <code>MEMORY_AND_DISK_SER</code>) spills the block to disk and reads it back on next use, at the cost of disk I/O; a memory-only level (<code>MEMORY_ONLY</code>) simply drops the block and recomputes it from source when needed again<sup class="footnote-ref"><a href="#fn2" id="fnref2:7">[2:7]</a></sup><sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:9">[1:9]</a></sup>.</p><img class="light-only" src="'+s+'" alt="A cached block moves to evicted under memory pressure or lost when an executor goes away, then either spills to disk under MEMORY_AND_DISK or recomputes from lineage under MEMORY_ONLY."><img class="dark-only" src="'+n+'" alt="A cached block moves to evicted under memory pressure or lost when an executor goes away, then either spills to disk under MEMORY_AND_DISK or recomputes from lineage under MEMORY_ONLY."><p>There&#39;s no hard limit on how many DataFrames can be cached simultaneously; the constraint is aggregate storage memory, not a count of objects<sup class="footnote-ref"><a href="#fn2" id="fnref2:8">[2:8]</a></sup>. That budget is fraction-based configuration translated into a live byte allowance, not a fixed absolute constant: <code>spark.memory.fraction</code> (default 0.6) sets the fraction of (heap minus a 300MB reserved region) used for the combined Execution+Storage region, and <code>spark.memory.storageFraction</code> (default 0.5) sets the portion of that region immune to eviction. Lowering either makes <a href="./bottleneck-spill.html">spills</a> and evictions more frequent<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:10">[1:10]</a></sup>.</p><h2 id="habits-that-keep-caching-effective" tabindex="-1">Habits that keep caching effective <a class="header-anchor" href="#habits-that-keep-caching-effective" aria-label="Permalink to &quot;Habits that keep caching effective&quot;">​</a></h2><p>A few habits keep caching effective despite how easily any of that goes wrong:</p><ul><li><strong>Materialize deliberately.</strong> Because caching is lazy, follow <code>cache()</code>/<code>persist()</code> with an action that forces a full pass (<code>count()</code> is the standard choice) rather than assuming the call alone did the work<sup class="footnote-ref"><a href="#fn2" id="fnref2:9">[2:9]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</li><li><strong>Match the storage level to the constraint you&#39;re solving.</strong> <code>cache()</code> / <code>MEMORY_AND_DISK</code> is a reasonable default: Spark keeps deserialized objects in memory and only serializes the overflow to disk. Reach for <code>MEMORY_AND_DISK_SER</code> when memory pressure is the binding constraint and you can afford the extra CPU cost of deserializing on read<sup class="footnote-ref"><a href="#fn2" id="fnref2:10">[2:10]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</li><li><strong>Protect cached data from dynamic allocation.</strong> If executors holding cached blocks might be reclaimed as idle, raise <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code> so they aren&#39;t pulled out from under the cache<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>, or use a replicated storage level (e.g., <code>MEMORY_AND_DISK_2</code>) so losing one executor doesn&#39;t lose the data<sup class="footnote-ref"><a href="#fn1" id="fnref1:11">[1:11]</a></sup>.</li><li><strong>Reach for checkpointing, not persisting, for genuine lineage truncation with fault tolerance</strong> across a long transformation chain, since it writes to a reliable external filesystem rather than relying on executor-local memory or disk<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. Avoid <code>localCheckpoint()</code> under dynamic allocation for the same reason caching is unsafe there: its data lives in ephemeral local executor storage that can vanish along with a reclaimed executor<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>. Also set a checkpoint directory via <code>SparkContext.setCheckpointDir</code> before calling <code>.checkpoint()</code>; without one, the call throws immediately rather than silently doing nothing<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>.</li><li><strong>Don&#39;t cache data that won&#39;t benefit:</strong> single-use datasets, datasets that don&#39;t fit in available memory, or datasets you only ever touch through partial actions (<code>take</code>, <code>limit</code>). In each of these cases the caching overhead isn&#39;t paid back<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:11">[2:11]</a></sup>.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a> <a href="#fnref1:10" class="footnote-backref">↩︎</a> <a href="#fnref1:11" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 — Optimizing and Tuning Spark Applications <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a> <a href="#fnref2:6" class="footnote-backref">↩︎</a> <a href="#fnref2:7" class="footnote-backref">↩︎</a> <a href="#fnref2:8" class="footnote-backref">↩︎</a> <a href="#fnref2:9" class="footnote-backref">↩︎</a> <a href="#fnref2:10" class="footnote-backref">↩︎</a> <a href="#fnref2:11" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 7 — Effective Transformations <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-caching" target="_blank" rel="noreferrer">Spark Tips: Caching</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 — Performance Tuning <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li></ol></section>',22)])])}const g=a(f,[["render",c]]);export{m as __pageData,g as default};

package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js

@@ -1 +0,0 @@
-import{_ as a,o,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/cache-lifecycle.rEOVYQNU.svg",n="/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg",m=JSON.parse('{"title":"Caching & Persistence","description":"","frontmatter":{"title":"Caching & Persistence"},"headers":[],"relativePath":"tuning-reference/caching.md","filePath":"tuning-reference/caching.md"}'),f={name:"tuning-reference/caching.md"};function c(i,e,l,h,d,p){return o(),t("div",null,[...e[0]||(e[0]=[r("",22)])])}const g=a(f,[["render",c]]);export{m as __pageData,g as default};

package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.js

@@ -1 +0,0 @@
-import{_ as o,o as t,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/container-memory.DIO0AnIm.svg",n="/docs/assets/container-memory.dark.CP-5zuCl.svg",m=JSON.parse('{"title":"Cluster Tuning","description":"","frontmatter":{"title":"Cluster Tuning"},"headers":[],"relativePath":"tuning-reference/cluster-config.md","filePath":"tuning-reference/cluster-config.md"}'),c={name:"tuning-reference/cluster-config.md"};function i(f,e,l,d,u,h){return t(),a("div",null,[...e[0]||(e[0]=[r('<h1 id="cluster-config" tabindex="-1">Cluster Tuning <a class="header-anchor" href="#cluster-config" aria-label="Permalink to &quot;Cluster Tuning {#cluster-config}&quot;">​</a></h1><h2 id="sizing-executors-and-containers" tabindex="-1">Sizing executors and containers <a class="header-anchor" href="#sizing-executors-and-containers" aria-label="Permalink to &quot;Sizing executors and containers&quot;">​</a></h2><p>Cluster tuning is the set of decisions that map an application&#39;s cores, memory, and executor count onto the underlying cluster (YARN or Kubernetes) plus a handful of related knobs (data locality, dynamic allocation, per-task CPU reservation) that all interact with that sizing decision.</p><p>The starting point is executor sizing. Working through a concrete example (a 10-node cluster with 16 cores and 64GB RAM per node), the derivation runs: assign 5 cores per executor; reserve about 1 core per node for Hadoop/YARN/OS daemons, leaving 15 usable cores per node (150 total); dividing by 5 cores per executor gives 30 executors, minus 1 reserved for the YARN ApplicationMaster (29); with 3 executors per node, memory per executor is 64GB / 3 ≈ 21GB, and after subtracting about 7% for YARN memory overhead that comes out to roughly 18GB usable, a recommended shape of 29 executors × 5 cores × 18GB<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Cloudera&#39;s own worked example, on a 6-node cluster with the same 16-core / 64GB-per-node hardware, lands on the same shape: <code>--num-executors 17 --executor-cores 5 --executor-memory 19G</code>, rather than one &quot;fat&quot; executor per node (<code>--num-executors 6 --executor-cores 15 --executor-memory 63G</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Both sources treat the resulting task count (executor-cores × num-executors) as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. <code>spark.executor.cores</code> is the config that controls concurrent tasks per executor; it defaults to 1 in YARN mode, or to all available cores on the worker in standalone mode<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>, and <code>spark.executor.memory</code> sets the JVM heap size per executor<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Independent of the sizing formula above, the Spark tuning guide recommends targeting 2–3 tasks per CPU core across the cluster<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p><p>On top of executor memory sits container memory. Spark computes the total container/pod allocation as <code>executorMemoryMiB + memoryOverheadMiB + memoryOffHeapMiB + pysparkMemToUseMiB</code>. On YARN this is what gets allocated for the container, on Kubernetes it becomes the pod memory limit<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. <code>spark.executor.memoryOverhead</code> itself defaults to <code>max(0.1 * executorMemory, 384MB)</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="'+s+'" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><img class="dark-only" src="'+n+'" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><blockquote><p><strong>PySpark:</strong> <code>spark.executor.pyspark.memory</code> is only added as its own container-memory term when it&#39;s set explicitly; otherwise PySpark&#39;s memory use is folded into the general overhead budget rather than tracked separately.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup></p></blockquote><p>On Kubernetes specifically, <code>spark.kubernetes.executor.request.cores</code> and <code>spark.kubernetes.executor.limit.cores</code> take priority over <code>spark.executor.cores</code> for the pod&#39;s CPU request/limit sent to the Kubernetes scheduler<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>, while <code>spark.executor.cores</code> remains the config Spark itself uses to size the number of concurrent task slots<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>Two more knobs round out the picture. <code>spark.locality.wait</code> (default <code>3s</code>) controls how long a task waits for a data-local placement before Spark gives up and schedules it less locally, stepping through the same wait across process-local → node-local → rack-local → any; each level can be overridden independently via <code>spark.locality.wait.process</code>, <code>.node</code>, and <code>.rack</code>, all of which default to the base wait when left unset<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. Dynamic allocation lets Spark add and remove executors as work changes, but it requires one of several supporting mechanisms: an external shuffle service, shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0), shuffle-block decommission, or the experimental sort-IO plugin<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Removing an executor can otherwise destroy shuffle state it&#39;s holding. Finally, <code>spark.task.cpus</code> (default <code>1</code>) sets how many cores each task reserves; the number of concurrent task slots per executor is derived jointly from it and <code>spark.executor.cores</code>, roughly <code>spark.executor.cores / spark.task.cpus</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><h2 id="spotting-a-bad-sizing-decision" tabindex="-1">Spotting a bad sizing decision <a class="header-anchor" href="#spotting-a-bad-sizing-decision" aria-label="Permalink to &quot;Spotting a bad sizing decision&quot;">​</a></h2><p>Getting that sizing wrong shows up in a handful of specific symptoms. The clearest sign of an under-sized executor layout is HDFS throughput dropping under load: &quot;HDFS client has trouble with tons of concurrent threads. It was observed that HDFS achieves full write throughput with ~5 tasks per executor&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The &quot;fat executor&quot; case (one executor per node, using all 16 cores) shows the failure mode directly: &quot;with all 16 cores per executor... HDFS throughput will hurt and it&#39;ll result in excessive garbage [collection]&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>On the memory side, an executor or pod that dies with no Spark-level error is a strong signal that off-heap memory was never added into the container budget. Enabling <code>spark.memory.offHeap.enabled=true</code> with <code>spark.memory.offHeap.size=1g</code> on top of an 8G executor with default overhead (819MB) means real usage is 8192+819+1024 = 10,035MB, while the container was only granted 8192+819 = 9,011MB. The executor gets killed by YARN, or OOMKilled by the Kubernetes kubelet, with no warning from Spark itself<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>.</p><p>With dynamic allocation on, unexplained shuffle recomputation is a detectable symptom of executors being reclaimed mid-shuffle: &quot;In the event of stragglers... dynamic allocation may remove an executor before the shuffle completes, in which case the shuffle files written by that executor must be recomputed unnecessarily&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Jobs that abort with a serialized-result-size error are hitting the <code>spark.driver.maxResultSize</code> guardrail rather than a driver heap exhaustion: &quot;Jobs will be aborted if the total size [of serialized action results] is above this limit&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. And on the resource-waste side, executors that are provisioned but never assigned work are a sign that dynamic allocation is targeting full parallelism against a workload made of many small tasks<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><h2 id="what-a-bad-sizing-decision-costs" tabindex="-1">What a bad sizing decision costs <a class="header-anchor" href="#what-a-bad-sizing-decision-costs" aria-label="Permalink to &quot;What a bad sizing decision costs&quot;">​</a></h2><p>Each symptom above carries a specific price tag. Task count (<code>executor-cores × num-executors</code>) is treated by both cited sources as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. Going the other way, cramming too many cores into one executor degrades HDFS throughput and drives up garbage collection<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</p><p>Container-memory misconfiguration matters because the failure is silent from Spark&#39;s point of view: off-heap memory that isn&#39;t accounted for in <code>spark.executor.memoryOverhead</code> causes the executor to actually use more memory than the container/pod was granted, so it gets killed externally (by YARN or the kubelet) with no Spark-level diagnostic to point at the real cause<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>Dynamic allocation&#39;s interaction with shuffle state matters because, before dynamic allocation existed, an executor exiting alongside its application meant all its state could be safely discarded; with dynamic allocation, the application keeps running after an executor is explicitly removed, so any later need for that executor&#39;s state forces a recompute<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. That is exactly why Spark needs &quot;a mechanism to decommission an executor gracefully by preserving its state before removing it&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>. Without shuffle tracking, an external shuffle service, or shuffle-block decommission enabled, dynamic allocation either can&#39;t be turned on at all, or, if it&#39;s active regardless, an executor holding unpreserved shuffle output that gets removed forces exactly that unnecessary recompute<sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</p><p>On the driver side, <code>spark.driver.maxResultSize</code> and <code>spark.driver.memory</code> are separate budgets that still interact: whether a high <code>maxResultSize</code> actually causes an out-of-memory error &quot;depends on spark.driver.memory and memory overhead of objects in JVM&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Raising one without considering the other doesn&#39;t fully protect the driver.</p><p>Finally, over-provisioning executors against a small-task workload wastes cluster resources: &quot;with small tasks this setting can waste a lot of resources due to executor allocation overhead, as some executor might not even do any work&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup>.</p><h2 id="sizing-the-cluster-correctly" tabindex="-1">Sizing the cluster correctly <a class="header-anchor" href="#sizing-the-cluster-correctly" aria-label="Permalink to &quot;Sizing the cluster correctly&quot;">​</a></h2><p>Avoiding those costs starts from a formula, not one fat executor per node. Apply the balanced-executor formula instead; the worked examples above give shapes like 29 × 5 × 18GB for a 10-node/16-core/64GB cluster, or 17 × 5 × 19G for a 6-node cluster with the same per-node hardware<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>. Keep <code>--executor-cores</code> at or below roughly 5 so HDFS client concurrency stays in the range where it sustains full write throughput<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>.</p><p>When enabling off-heap memory, account for <code>spark.memory.offHeap.size</code> in the container/pod memory budget explicitly; <code>spark.executor.memoryOverhead</code>&#39;s default of <code>max(0.1 * executorMemory, 384MB)</code> does not include it<sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>. If you&#39;re still referencing the older <code>spark.yarn.executor.memoryOverhead</code> name, note it was removed in Spark 3.0<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><p>On Kubernetes, remember that <code>spark.kubernetes.executor.request.cores</code> and <code>.limit.cores</code> govern the pod&#39;s CPU request/limit and take priority over <code>spark.executor.cores</code> for that purpose<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>, while <code>spark.executor.cores</code> separately governs Spark&#39;s own task-slot count<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. The two need to be reasoned about together rather than assumed to be redundant.</p><p>Tune <code>spark.locality.wait</code> (and its per-level overrides) upward when tasks are long-running and locality is poor, since the default is tuned for typical workloads<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>; set <code>spark.locality.wait.node</code> to <code>0</code> to skip straight to rack locality when node locality isn&#39;t achievable<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>.</p><p>Turn on shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0) or the external shuffle service so dynamic allocation can reclaim executors without losing shuffle output or forcing recomputation<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup>. The external shuffle service additionally lets persisted RDD blocks survive executor removal when <code>spark.shuffle.service.fetch.rdd.enabled</code> is set, and executors holding cached blocks are, by default, never removed at all, tunable via <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup>.</p><p>Set <code>spark.driver.maxResultSize</code> as a guardrail on serialized action-result size, and size <code>spark.driver.memory</code> with it in mind rather than in isolation<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Use <code>spark.dynamicAllocation.executorAllocationRatio</code> (default <code>1.0</code>, added in 2.4.0) to scale down over-allocation when tasks are small; for example, a value of <code>0.5</code> halves the target executor count dynamic allocation would otherwise compute<sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>. Raise <code>spark.task.cpus</code> above its default of <code>1</code> when a task needs more than one core; doing so proportionally reduces concurrent task slots per executor (<code>spark.executor.cores / spark.task.cpus</code>)<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><h2 id="bottleneck-autoscaling-churn" tabindex="-1">Autoscaling churn <a class="header-anchor" href="#bottleneck-autoscaling-churn" aria-label="Permalink to &quot;Autoscaling churn {#bottleneck-autoscaling-churn}&quot;">​</a></h2><p><span class="tag">CHRN</span> <span class="tag">EXPERIMENTAL</span></p><p>Dynamic allocation adds executors once tasks back up and frees them once they go idle<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup>. Churn is a narrower failure mode inside that same mechanism: executors that get stood up and torn down again before they&#39;ve done much useful work, cycling through provisioning and JVM startup cost repeatedly instead of running tasks.</p><h3 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>An executor counts as short-lived when its lifetime, from its <code>ExecutorAdded</code> event to either its <code>ExecutorRemoved</code> event or the end of the run if it was never removed, is under 2 minutes. This is evaluated once a run has added at least 5 executors, a floor that keeps a two- or three-executor job from registering on ordinary scale-down.</p><table tabindex="0"><thead><tr><th>Signal</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Share of added executors that are short-lived (&lt; 2 min lifetime)</td><td>&gt; 30%</td><td>&gt; 60%</td></tr></tbody></table><p>The 2-minute lifetime cutoff and the 30%/60% split are unvalidated heuristics rather than figures published in Spark&#39;s own documentation or benchmarks, so treat a finding here as a prompt to look at the executor timeline rather than a calibrated verdict.</p><h3 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>Every short-lived executor pays the same provisioning and JVM-startup cost as a long-lived one, but returns little task work in exchange. A run that keeps flapping between scaling up and down spends more of its wall-clock time paying that repeated overhead than one that settles into a stable executor count.</p><h3 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><ul><li>Raise <code>spark.dynamicAllocation.executorIdleTimeout</code> (default 60s<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>) so an executor survives a brief lull instead of being released the moment it goes idle, only to be requested again shortly after.</li><li>Widen the gap between <code>spark.dynamicAllocation.minExecutors</code> and <code>.maxExecutors</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup>: bounds set too close together force Spark to repeatedly add and remove executors to track small fluctuations in the task backlog instead of settling into a stable range.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores and Memory for a Spark Application</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Dynamic Resource Allocation</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li></ol></section>',41)])])}const k=o(c,[["render",i]]);export{m as __pageData,k as default};

package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.lean.js

@@ -1 +0,0 @@
-import{_ as o,o as t,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/container-memory.DIO0AnIm.svg",n="/docs/assets/container-memory.dark.CP-5zuCl.svg",m=JSON.parse('{"title":"Cluster Tuning","description":"","frontmatter":{"title":"Cluster Tuning"},"headers":[],"relativePath":"tuning-reference/cluster-config.md","filePath":"tuning-reference/cluster-config.md"}'),c={name:"tuning-reference/cluster-config.md"};function i(f,e,l,d,u,h){return t(),a("div",null,[...e[0]||(e[0]=[r("",41)])])}const k=o(c,[["render",i]]);export{m as __pageData,k as default};